From b0414a6089632acbd4a9ac23155e41c76192b9b0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 01:42:17 -0700 Subject: [PATCH 01/14] Add the kana Script members and the effective_script kana license (#272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 1 of the #272 Japanese-support plan: classification only, no order/segmentation defaults change. Script gains HIRAGANA (U+3040- U+309F) and KATAKANA (U+30A0-U+30FF, which includes the prolonged mark and the U+30FB middle dot). _vocab.effective_script extends single_script with the kana license from the 2026-07-29 amendment: a mixed Han/hiragana/katakana token is Japanese and resolves to the HIRAGANA carrier entry; pure-katakana keeps single_script's own answer since nothing defaults on it. Updated two pre-existing pins that this table change broke: the single_script test asserting kana was unclassified (now KATAKANA), and test_regex_sync's #271 differential-rule sync check, which now scopes its expected spans to the HAN/HANGUL scripts #271 actually governs rather than every _SCRIPT_RANGES entry, since #272 grows the same table for an unrelated reason. Quality-review fix-first pass, folded into this same commit: both classifiers now run on an NFC-normalized copy of the text, never the raw input. NFD decomposes precomposed katakana onto a base character plus a combining voiced/semi-voiced sound mark (U+3099/U+309A), which sit in the HIRAGANA block rather than katakana's -- classifying raw NFD text could therefore see a pure-katakana token as Han-free but kana-mixed and wrongly hand it the kana license (HIRAGANA) instead of declining it (KATAKANA). Separately, NFD decomposes Hangul syllables onto bare jamo (U+1100-U+11FF), entirely outside the HANGUL range, so raw NFD Korean input missed the shipped family-first order rule altogether and silently fell back to the positional default -- a live gap in #294's shipped behavior, not merely a #272 nicety. Both directions are demonstrated in tests/v2/pipeline/test_vocab.py by simulating the pre-fix classifier against NFD input built with unicodedata.normalize() (never pasted as decomposed literals): the pre-fix classifier returns None for both an NFD katakana and an NFD hangul string, and would have returned HIRAGANA (wrongly) for the raw NFD katakana case under a naive license check. Matching (is_initial, suffix lookups, etc.) deliberately stays on raw text elsewhere in the module -- NFD only ever costs a match there, never a wrong one, so that asymmetry is safe and unchanged. The normalized copy is used for classification only; token text and rendered spans are exactly what the caller wrote, still NFD if that's what came in (see the new test_nfd_korean_input_still_reads_family_first in test_parser.py, which compares NFC-normalized field values since the rendered family/ given text itself stays NFD). Also folded in from the same review: reworded the "no supplementary- plane kana to chase" claim (false -- 311 assigned codepoints exist; none are worth chasing, since they're archaic/phonetic-extension forms no modern name uses); added the halfwidth-kana exclusion rationale and the block-vs-UAX#24 clause to the range-table comment; corrected "kana-only" to "katakana-only" in effective_script's docstring (さくらエミ is kana-only and licensed); added a forward-pointer from single_script to effective_script; moved _JA_SCRIPTS/_JA_PATTERN up beside _SCRIPT_PATTERNS and changed _JA_SCRIPTS from a sorted frozenset to a plain table-ordered tuple (nothing uses membership); made HIRAGANA's Policy docstring self-contained; folded the temporal-named Script-enum test into test_script_values_are_the_public_names; and trimmed the Han/Hangul-duplicate assertions out of the kana single_script test (renamed test_kana_singles_classify). Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_vocab.py | 120 ++++++++++++++++++++++++++++---- nameparser/_policy.py | 9 +++ tests/v2/pipeline/test_vocab.py | 70 ++++++++++++++++++- tests/v2/test_parser.py | 16 +++++ tests/v2/test_policy.py | 1 + tests/v2/test_regex_sync.py | 22 ++++-- 6 files changed, 216 insertions(+), 22 deletions(-) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index f43a128..b25cab4 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -9,6 +9,7 @@ from __future__ import annotations import re +import unicodedata from nameparser._lexicon import Lexicon, _normalize from nameparser._policy import Script @@ -42,17 +43,37 @@ # 𠮷, U+20BB7), so leaving them out silently mis-orders those names; # unassigned gaps inside the span are harmless, since no real name # contains an unassigned codepoint. HANGUL: precomposed syllables -# only -- modern Korean text never writes names as bare jamo. Kana is -# DELIBERATELY absent: a kana token identifies Japanese, whose -# conventions are #272's segmenter, not this table's. The ranges -# below must stay mutually disjoint: single_script returns the FIRST -# covering entry (dict iteration order), so an overlapping future -# script (e.g. a ja entry that also covers Han) would make the result -# order-dependent instead of well-defined. +# only -- modern Korean text never writes names as bare jamo. +# HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There +# IS a supplementary-plane kana repertoire (Kana Supplement, Kana +# Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, 311 assigned +# codepoints) but none of it is WORTH chasing the way Han's astral +# block is: those codepoints are hentaigana and other archaic/ +# phonetic-extension forms no modern Japanese name uses, unlike +# supplementary Han, which real surnames genuinely need. Halfwidth +# kana (U+FF65-U+FF9D) is likewise deliberately excluded -- legacy +# bank/CSV data uses it, but it is a separate normalization problem; +# Task 2b's separator handling only touches the halfwidth DOT +# (U+FF65), not the rest of that block. This table classifies by +# Unicode BLOCK, not the UAX #24 Script property: U+30A0, U+30FB +# (the middle dot), and U+30FC (the prolonged sound mark) all carry +# Script=Common under UAX #24, and the combining kana voicing marks +# U+3099-U+309C are Common/Inherited -- yet every one of them is +# needed here, and block membership, not the Script property, is +# what puts them in range. The katakana block's upper end (U+30FF) +# including the middle dot U+30FB is load-bearing for +# effective_script's kana license below (see its docstring) -- a +# later task turns U+30FB into a tokenize-level separator, but until +# then it classifies as ordinary katakana. The ranges below must stay +# mutually disjoint: single_script returns the FIRST covering entry +# (dict iteration order), so an overlapping future script would make +# the result order-dependent instead of well-defined. _SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { Script.HAN: ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)), Script.HANGUL: ((0xAC00, 0xD7A3),), + Script.HIRAGANA: ((0x3040, 0x309F),), + Script.KATAKANA: ((0x30A0, 0x30FF),), } # Derived, never hand-written: one character class per script, in the @@ -65,6 +86,21 @@ for script, ranges in _SCRIPT_RANGES.items() } +#: The Japanese repertoire: the union effective_script's kana license +#: quantifies over. A tuple in the table's own order (HAN, then +#: HIRAGANA, then KATAKANA -- HANGUL simply omitted, the relative +#: order of the rest is unchanged), not a frozenset: nothing consults +#: membership today, only iterates to build the pattern below, so +#: there is nothing set-ness would buy; a later task that needs +#: membership can convert it then. +_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) +_JA_PATTERN = re.compile( + "[" + + "".join(f"\\U{lo:08x}-\\U{hi:08x}" + for s in _JA_SCRIPTS + for lo, hi in _SCRIPT_RANGES[s]) + + "]+") + def is_initial(text: str) -> bool: """'A.' / 'j.' / bare capital -- v1's is_an_initial.""" @@ -162,18 +198,72 @@ def period_joined_vocab(text: str, lexicon: Lexicon) -> str | None: return None +def _normalized_for_script(text: str) -> str | None: + """The guard AND the NFC normalization single_script and + effective_script's license path both need, single-sourced so they + cannot drift: None for the two shapes neither ever classifies + (empty, and the common all-ASCII Latin token -- skipped before + normalizing, since ASCII is already NFC and every _SCRIPT_RANGES + entry is non-ASCII regardless), else an NFC-normalized copy. + + NFC, not raw: NFD input decomposes precomposed katakana onto a + base character plus a COMBINING mark (U+3099/U+309A, which sit in + the HIRAGANA block, not katakana's), so classifying raw NFD text + can hand a pure-katakana token the kana license by accident; NFD + also decomposes Hangul syllables onto bare jamo (U+1100-U+11FF), + entirely outside the HANGUL range, so raw NFD Korean input misses + the shipped family-first order rule rather than merely misfiring. + Normalizing first fixes both. This is classification-only and + read-only: the returned copy is never what gets tokenized, so + token text and spans stay exactly what the caller wrote. + + MATCHING (is_initial, suffix lookups, etc.) deliberately stays on + raw text elsewhere in this module -- unlike script classification, + NFD only ever costs a match there (a suffix word written NFD fails + to match its NFC vocabulary entry), never wrong-matches, so the + asymmetry is safe: one direction needs a fix, the other doesn't. + """ + if not text or text.isascii(): + return None + return unicodedata.normalize("NFC", text) + + def single_script(text: str) -> Script | None: """The one Script whose ranges cover EVERY char of `text`, else None (mixed-script text has no well-defined convention to apply; - the caller falls back to the positional default).""" - if not text: - return None # the + below needs one char; "" belongs to no script - if text.isascii(): - # every _SCRIPT_RANGES entry is non-ASCII (lowest today is - # U+3400): skip the patterns for the overwhelmingly common - # Latin token (the _tokenize._ignorable ASCII-floor precedent) + the caller falls back to the positional default). Classifies an + NFC-normalized copy of `text` -- see _normalized_for_script. + Callers wanting the kana-mixed license (a kanji+kana composite + resolving to HIRAGANA) want effective_script, not this function.""" + normalized = _normalized_for_script(text) + if normalized is None: return None for script, pattern in _SCRIPT_PATTERNS.items(): - if pattern.fullmatch(text): + if pattern.fullmatch(normalized): return script return None + + +def effective_script(text: str) -> Script | None: + """single_script, extended by the kana license (#272 amendment): + a MIXED token wholly within Han∪hiragana∪katakana is Japanese -- + it necessarily contains kana (pure Han is not mixed), cannot be + Chinese, and is not a foreign transcription (those are + katakana-only: マイケル has no kanji, but さくらエミ -- hiragana + plus katakana -- is kana-only AND licensed) -- and resolves to the + HIRAGANA carrier entry. Pure-katakana stays KATAKANA + (single_script's answer): a lone katakana token is predominantly a + transcribed foreign name, so nothing defaults on it.""" + script = single_script(text) + if script is not None: + return script + # normalized is None for both shapes _JA_PATTERN could never match + # anyway (empty text, or the all-ASCII text single_script's fast + # path already ruled out) -- real work, not a leftover "if text" + # guard: unlike the pre-NFC version, None here also covers the + # ASCII case, which single_script's own empty check alone would + # not. + normalized = _normalized_for_script(text) + if normalized is not None and _JA_PATTERN.fullmatch(normalized): + return Script.HIRAGANA + return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 8a01ab2..05daaac 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -45,6 +45,15 @@ class Script(StrEnum): HAN = "han" #: Korean Hangul (precomposed syllables). Unambiguously Korean. HANGUL = "hangul" + #: Japanese hiragana. Never transcribes foreign names, so a mixed + #: kanji+kana token (高橋みなみ) is Japanese and resolves HERE -- + #: this member is the carrier key in script_orders/segment_scripts. + HIRAGANA = "hiragana" + #: Japanese katakana. A PURE-katakana token is predominantly a + #: transcribed foreign name in its original order (マイケル), so + #: no default behavior keys on this member; it exists so the + #: classifier can name what it deliberately declines. + KATAKANA = "katakana" # Order-spec constants (#270). Each reads as its contents because roles diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 82ef629..ff62311 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -1,7 +1,9 @@ +import unicodedata + from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( - _SCRIPT_RANGES, is_initial, is_suffix_lenient, is_suffix_strict, - single_script, + _SCRIPT_RANGES, effective_script, is_initial, is_suffix_lenient, + is_suffix_strict, single_script, ) from nameparser._policy import Script @@ -51,7 +53,10 @@ def test_single_script_requires_every_char_in_one_script() -> None: assert single_script("Smith") is None assert single_script("毛zedong") is None # mixed assert single_script("毛김") is None # mixed CJK - assert single_script("イチロー") is None # kana: not HAN + # kana classifies as its own script (#272), not HAN -- was None + # before kana had a table entry; single_script's job is telling + # kana apart from Han, not lumping the two together + assert single_script("イチロー") is Script.KATAKANA def test_single_script_range_edges() -> None: @@ -82,3 +87,62 @@ def test_no_script_range_reaches_ascii() -> None: # it fails here rather than silently going unclassified. assert all(lo >= 0x80 for ranges in _SCRIPT_RANGES.values() for lo, _ in ranges) + + +def test_kana_singles_classify() -> None: + assert single_script("みなみ") is Script.HIRAGANA + assert single_script("エミ") is Script.KATAKANA + assert single_script("ー") is Script.KATAKANA # prolonged mark, in-block + + +def test_effective_script_kana_license() -> None: + # pure single-script tokens pass through unchanged + assert effective_script("山田") is Script.HAN + assert effective_script("みなみ") is Script.HIRAGANA + assert effective_script("マイケル") is Script.KATAKANA + # the license: a MIXED token wholly in Han∪kana is Japanese and + # resolves to the HIRAGANA carrier entry + assert effective_script("高橋みなみ") is Script.HIRAGANA # kanji+hira + assert effective_script("山田エミ") is Script.HIRAGANA # kanji+kata + assert effective_script("さくらエミ") is Script.HIRAGANA # hira+kata + # outside the license: anything beyond the JA repertoire + assert effective_script("毛김") is None + assert effective_script("山田x") is None + # NOT None: U+30FB sits INSIDE the katakana block (verified by + # codepoint), so this stays a PURE-katakana token, same as + # "マイケル" above -- the license only ever fires on a MIXED + # token, and this one isn't mixed. It has no order-default entry + # (DEFAULT_SCRIPT_ORDERS carries no KATAKANA key), which is where + # "declines the license" actually shows up. A later task turns + # U+30FB into a tokenize-level separator, so this string arrives + # at effective_script as two tokens instead of one. + assert effective_script("マイケル・ジャクソン") is Script.KATAKANA + assert effective_script("") is None + + +def test_nfd_katakana_still_classifies_and_declines_the_license() -> None: + # Built via normalize(), never pasted as decomposed literals -- + # NFD "ガガ" is base katakana カ + COMBINING VOICED SOUND MARK + # (U+3099) twice; U+3099 sits in the HIRAGANA block, not + # katakana's, so classifying raw NFD text would see one char from + # each block and either call it mixed (single_script: None) or + # wrongly grant the kana license (effective_script: HIRAGANA) for + # what is really one pure-katakana token. NFC-normalizing first + # (the #272 amendment's NFC decision) recomposes it back to ガガ, + # which reads as ordinary katakana either way -- the license + # still correctly declines, because this token isn't mixed. + nfd = unicodedata.normalize("NFD", "ガガ") + assert nfd != "ガガ" # sanity: confirms the decomposition actually ran + assert single_script(nfd) is Script.KATAKANA + assert effective_script(nfd) is Script.KATAKANA + + +def test_nfd_hangul_still_classifies() -> None: + # NFD decomposes each precomposed syllable onto 2-3 jamo + # (U+1100-U+11FF), entirely outside the HANGUL range -- raw NFD + # input would silently miss the shipped family-first order rule + # rather than merely misclassify. Built via normalize(), not + # pasted decomposed literals, same reason as above. + nfd = unicodedata.normalize("NFD", "김민준") + assert nfd != "김민준" # sanity: confirms the decomposition actually ran + assert single_script(nfd) is Script.HANGUL diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index a261264..9948019 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -1,5 +1,6 @@ import dataclasses import pickle +import unicodedata import pytest @@ -451,6 +452,21 @@ def test_wholly_cjk_names_read_family_first_by_default() -> None: assert parse("毛泽东").family == "毛泽东" +def test_nfd_korean_input_still_reads_family_first() -> None: + # fix(#271) classification, landed via the #272 NFC-classification + # amendment: NFD decomposes each Hangul syllable onto bare jamo, + # entirely outside the HANGUL range, so raw NFD input used to miss + # script_orders' family-first rule and fall back to the positional + # default -- a live gap in #294's shipped behavior until + # single_script started classifying an NFC-normalized copy. + n = parse(unicodedata.normalize("NFD", "김 민준")) + # classification-only: the rendered text is exactly what was + # typed (still NFD, spans untouched), so compare NFC-normalized -- + # the point under test is the ORDER (family first), not encoding + assert (unicodedata.normalize("NFC", n.family), + unicodedata.normalize("NFC", n.given)) == ("김", "민준") + + def test_latin_names_are_untouched_by_script_orders() -> None: n = parse("John Smith") assert (n.given, n.family) == ("John", "Smith") diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index d8fa542..7e1cb8e 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -63,6 +63,7 @@ def test_name_order_rejects_plain_string_tuples() -> None: def test_script_values_are_the_public_names() -> None: assert Script.HAN == "han" and Script.HANGUL == "hangul" + assert Script.HIRAGANA == "hiragana" and Script.KATAKANA == "katakana" def test_patronymic_rules_coerce_and_reject() -> None: diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 68b7f6f..f203088 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -23,6 +23,7 @@ from nameparser.config import regexes as _config from nameparser._pipeline import _assign, _post_rules, _tokenize, _vocab +from nameparser._policy import Script from nameparser import _render @@ -114,6 +115,9 @@ def test_initial_copies_agree_with_each_other_and_config() -> None: ("_tokenize", "_BIDI"): None, # re_bidi, not a REGEXES key # Mirrors _pipeline._state.COMMA_CHARS, not nameparser.config ("_render", "_COMMA_CHAR"): None, + # #272: derived from _SCRIPT_RANGES itself (like _SCRIPT_PATTERNS), + # not hand-copied from anywhere -- no config counterpart to pin. + ("_vocab", "_JA_PATTERN"): None, } _MODULES = {"_assign": _assign, "_post_rules": _post_rules, @@ -176,8 +180,17 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: Han's astral block is out of scope on both sides. The rule omits it deliberately -- no corpus name reaches it, see the comment there -- so the comparison runs over the BMP spans only, and a new - BMP script added to _SCRIPT_RANGES still fails here until the rule - covers it. + BMP script added to _SCRIPT_RANGES for the SAME #271 behavior + (family-first order / hangul segmentation) still fails here until + the rule covers it. + + #272 added HIRAGANA/KATAKANA to _SCRIPT_RANGES for an unrelated + reason (kana classification for effective_script's license, not + the order-flip or segmentation #271's diff rule explains -- no + default keys on either kana member), so the comparison is scoped + to the #271 scripts by name rather than every table entry; a + future script added for #271's own reason still must extend both + sides, same as before. """ toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_changes.toml") @@ -191,8 +204,9 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", matched[0]["name_regex"])} expected = {span - for spans in _vocab._SCRIPT_RANGES.values() - for span in spans if span[1] <= 0xFFFF} + for script in (Script.HAN, Script.HANGUL) + for span in _vocab._SCRIPT_RANGES[script] + if span[1] <= 0xFFFF} assert declared == expected, ( f"{toml_path.name}'s #271 name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") From cb4346d58d9f6d93693c11a4643db0dbb5562e95 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 02:18:30 -0700 Subject: [PATCH 02/14] Read kana-licensed Japanese names family-first by default (#272) Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_assign.py | 55 ++++++++++++++------- nameparser/_pipeline/_vocab.py | 83 ++++++++++++++++++++++++-------- nameparser/_policy.py | 32 ++++++++---- tests/v2/cases.py | 6 +-- tests/v2/pipeline/test_assign.py | 16 ++++++ tests/v2/pipeline/test_vocab.py | 26 +++++++++- tests/v2/test_facade.py | 5 ++ tests/v2/test_parser.py | 27 +++++++++-- tests/v2/test_policy.py | 10 +++- 9 files changed, 204 insertions(+), 56 deletions(-) diff --git a/nameparser/_pipeline/_assign.py b/nameparser/_pipeline/_assign.py index 5d5fa89..3672341 100644 --- a/nameparser/_pipeline/_assign.py +++ b/nameparser/_pipeline/_assign.py @@ -3,9 +3,10 @@ Consumes: pieces + piece_tags (grouped), segments, structure, tokens. Produces: tokens with roles set on every main-stream token. Reads: Policy.name_order (#270) and Policy.script_orders (#271, which -overrides it when every name piece is written wholly in one script); -token/piece tags; Lexicon only through tags already applied by classify -(plus the leading-title period rule). +overrides it when every name piece is written wholly in one script, or +in the Han/Hiragana/Katakana repertoire the #272 kana license shares +across pieces); token/piece tags; Lexicon only through tags already +applied by classify (plus the leading-title period rule). Ports v1's assignment loops. NO_COMMA (per name_order): leading title pieces chain while no given-position name has been seen @@ -29,7 +30,9 @@ import dataclasses import re -from nameparser._pipeline._vocab import is_suffix_lenient, single_script +from nameparser._pipeline._vocab import ( + effective_script, is_suffix_lenient, resolve_script_set, +) from nameparser._pipeline._group import ( _is_suffix_piece, _is_title_piece, ) @@ -85,27 +88,43 @@ def _effective_order(policy: Policy, order governs the positional read; anything else -- Latin, mixed scripts, no entry -- falls back to name_order. Piece-level, after title/suffix peeling: 'Dr. 毛泽东' is a wholly-Han NAME under a - Latin title.""" + Latin title. Kana-licensed tokens (高橋みなみ, #272) resolve to + HIRAGANA the same way a wholly-Han or wholly-Hangul token resolves + to its own script -- and so does a kana-licensed NAME split across + separately single-script PIECES ('高橋 みなみ', Han piece plus + Hiragana piece): resolve_script_set generalizes the license from + one token's characters to the whole found-script set below, which + is why Han+Hangul ('毛 김') still declines even though both + individually read family-first -- the license is specific to the + Han/Hiragana/Katakana repertoire, not "the entries happen to + agree". + + Naming note, since the two are easy to conflate: THIS function + resolves the ORDER for a whole name; `_vocab.effective_script` + resolves the SCRIPT for a single token. This function calls that + one per token below. + """ if not policy.script_orders: return policy.name_order - # ONE script for the whole name, not "the entries all agree": two - # scripts that both read family-first still fall back, because a - # Han+Hangul name is not written in either tradition. Per token - # rather than per joined piece -- WorkToken text is never empty, so - # "the piece is wholly one script" is "every token in it is". - found: Script | None = None + # Collect every token's script rather than comparing pairwise as + # tokens are seen: the kana license needs the WHOLE set (a Han + # piece and a Hiragana piece only license together, never one at a + # time), so resolution is deferred to resolve_script_set below. + found: set[Script] = set() for piece in pieces: for i in piece: - script = single_script(tokens[i].text) + script = effective_script(tokens[i].text) if script is None: # Latin, mixed, or a script with no entry: never a key return policy.name_order - if found is None: - found = script - elif script is not found: - return policy.name_order - return next((order for s, order in policy.script_orders if s is found), - policy.name_order) + found.add(script) + resolved = resolve_script_set(found) + if resolved is None: + # e.g. Han+Hangul: two scripts, neither the kana license's + # Han/Hiragana/Katakana repertoire -- no single tradition + return policy.name_order + return next((order for s, order in policy.script_orders + if s is resolved), policy.name_order) def _name_positions(order: tuple[Role, Role, Role], diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index b25cab4..27fa505 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -10,6 +10,7 @@ import re import unicodedata +from collections.abc import Iterable from nameparser._lexicon import Lexicon, _normalize from nameparser._policy import Script @@ -50,19 +51,22 @@ # codepoints) but none of it is WORTH chasing the way Han's astral # block is: those codepoints are hentaigana and other archaic/ # phonetic-extension forms no modern Japanese name uses, unlike -# supplementary Han, which real surnames genuinely need. Halfwidth -# kana (U+FF65-U+FF9D) is likewise deliberately excluded -- legacy -# bank/CSV data uses it, but it is a separate normalization problem; -# Task 2b's separator handling only touches the halfwidth DOT -# (U+FF65), not the rest of that block. This table classifies by -# Unicode BLOCK, not the UAX #24 Script property: U+30A0, U+30FB -# (the middle dot), and U+30FC (the prolonged sound mark) all carry -# Script=Common under UAX #24, and the combining kana voicing marks -# U+3099-U+309C are Common/Inherited -- yet every one of them is -# needed here, and block membership, not the Script property, is -# what puts them in range. The katakana block's upper end (U+30FF) -# including the middle dot U+30FB is load-bearing for -# effective_script's kana license below (see its docstring) -- a +# supplementary Han, which real surnames genuinely need. The Katakana +# Phonetic Extensions block (U+31F0-U+31FF, 16 small katakana for Ainu +# transcription) is excluded for the same reason -- no modern Japanese +# personal name uses them. Halfwidth kana (U+FF65-U+FF9F, including +# the voiced/semi-voiced sound marks U+FF9E/U+FF9F) is likewise +# deliberately excluded -- legacy bank/CSV data uses it, but it is a +# separate normalization problem; Task 2b's separator handling only +# touches the halfwidth DOT (U+FF65), not the rest of that block. +# This table classifies by Unicode BLOCK, not the UAX #24 Script +# property: U+30A0, U+30FB (the middle dot), and U+30FC (the +# prolonged sound mark) all carry Script=Common under UAX #24, and the +# combining kana voicing marks U+3099-U+309C are Common/Inherited -- +# yet every one of them is needed here, and block membership, not the +# Script property, is what puts them in range. The katakana block's +# upper end (U+30FF) including the middle dot U+30FB is load-bearing +# for effective_script's kana license below (see its docstring) -- a # later task turns U+30FB into a tokenize-level separator, but until # then it classifies as ordinary katakana. The ranges below must stay # mutually disjoint: single_script returns the FIRST covering entry @@ -87,13 +91,14 @@ } #: The Japanese repertoire: the union effective_script's kana license -#: quantifies over. A tuple in the table's own order (HAN, then -#: HIRAGANA, then KATAKANA -- HANGUL simply omitted, the relative -#: order of the rest is unchanged), not a frozenset: nothing consults -#: membership today, only iterates to build the pattern below, so -#: there is nothing set-ness would buy; a later task that needs -#: membership can convert it then. -_JA_SCRIPTS = (Script.HAN, Script.HIRAGANA, Script.KATAKANA) +#: quantifies over -- HAN, HIRAGANA, KATAKANA (HANGUL simply omitted). +#: A frozenset, not the tuple this started as: resolve_script_set +#: below is the "later task that needs membership" the tuple's +#: original comment anticipated. Membership doesn't care about order, +#: and the pattern built below doesn't either (a regex character +#: class matches the same set regardless of the order its ranges are +#: written in). +_JA_SCRIPTS = frozenset({Script.HAN, Script.HIRAGANA, Script.KATAKANA}) _JA_PATTERN = re.compile( "[" + "".join(f"\\U{lo:08x}-\\U{hi:08x}" @@ -257,6 +262,15 @@ def effective_script(text: str) -> Script | None: script = single_script(text) if script is not None: return script + # Renormalizes text that single_script (above) already normalized + # once: deliberately NOT hoisted into a shared `_classify(normalized)` + # helper. The second call only runs on the fall-through path (a + # token single_script could not classify at all, i.e. genuinely + # mixed-script or empty/ASCII), never on the common single-script + # hit above, so it is a quick re-check on already-NFC text, not + # measured work -- the per-char-vs-regex history in this module's + # header is what a real cost here would look like, and this isn't + # it. # normalized is None for both shapes _JA_PATTERN could never match # anyway (empty text, or the all-ASCII text single_script's fast # path already ruled out) -- real work, not a leftover "if text" @@ -267,3 +281,32 @@ def effective_script(text: str) -> Script | None: if normalized is not None and _JA_PATTERN.fullmatch(normalized): return Script.HIRAGANA return None + + +def resolve_script_set(scripts: Iterable[Script]) -> Script | None: + """Generalizes effective_script's kana license from one token's + CHARACTERS to a whole name's PIECES (#272): `scripts` is the + effective_script of every name token, already resolved + individually -- '高橋' (Han) and 'みなみ' (Hiragana) are two + separately single-script pieces (split by a space, not mixed + within one token), but together are exactly the repertoire + effective_script licenses inside a single token (高橋みなみ). A + single distinct script is returned as-is (the ordinary case, + including a lone wholly-katakana name, which callers key with no + table entry); more than one collapses to the HIRAGANA carrier + when confined to Han/Hiragana/Katakana, the same set + effective_script's license tests; any other mix (Han+Hangul, or + no scripts at all -- an empty `scripts`) returns None -- the + caller's cue to fall back to the positional default, exactly like + effective_script's own None. A non-None result reports what was + FOUND, not that a license fired: callers wanting to know whether + the kana license specifically was the reason must compare the + result against a specific Script (e.g. `is Script.HIRAGANA`), not + just its truthiness -- a lone wholly-Han name also returns + non-None here, licensing nothing.""" + found = frozenset(scripts) + if len(found) <= 1: + return next(iter(found), None) + if found <= _JA_SCRIPTS: + return Script.HIRAGANA + return None diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 05daaac..42ed2d8 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -109,17 +109,27 @@ def _order_repr(value: tuple[Role, ...]) -> str: "parser_for(locales.RU) / locales.TR_AZ)" ) -#: Policy.script_orders' default: wholly-Han and wholly-Hangul names -#: read family-first. Public and named so opting out or extending -#: reads against a documented value (the DEFAULT_NICKNAME_DELIMITERS -#: precedent). The HAN entry is safe WITHOUT knowing Chinese from -#: Japanese: both write family-first in native script -- the -#: languages differ, the convention doesn't. Canonical form: sorted -#: (Script, order) pairs, matching the field's storage. +#: Policy.script_orders' default: wholly-Han, wholly-Hangul, and +#: kana-licensed names read family-first. Public and named so opting +#: out or extending reads against a documented value (the +#: DEFAULT_NICKNAME_DELIMITERS precedent). The HAN entry is safe +#: WITHOUT knowing Chinese from Japanese: both write family-first in +#: native script -- the languages differ, the convention doesn't. +#: HIRAGANA joins by the same rule as HANGUL (the kana license, +#: amendment 2026-07-29): a mixed Han-and-kana token cannot be +#: Chinese (it contains kana) and is not a foreign transcription +#: (transcriptions are katakana-only), so it is Japanese, written +#: family-first -- another default change in a minor, release-log- +#: classified fix, #294's mechanism. KATAKANA is deliberately absent: +#: a PURE-katakana token is predominantly a transcribed foreign name +#: kept in its source (usually given-first) order, so nothing should +#: default on it. Canonical form: sorted (Script, order) pairs, +#: matching the field's storage. DEFAULT_SCRIPT_ORDERS: tuple[ tuple[Script, tuple[Role, Role, Role]], ...] = ( (Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST), + (Script.HIRAGANA, FAMILY_FIRST), ) #: Policy.nickname_delimiters' default. Public and named so @@ -408,9 +418,11 @@ class Policy: #: ("John Smith, Jr.") leaves name_order governing the name part. name_order: tuple[Role, Role, Role] = GIVEN_FIRST #: Per-script overrides of name_order (#271), consulted when every - #: name piece is written wholly in one script: {Script: order} - #: (constructor accepts a mapping; stored as sorted pairs). The - #: default reads wholly-Han/Hangul names family-first -- see + #: name piece is written wholly in one script, or in the + #: Han/Hiragana/Katakana repertoire the #272 kana license shares + #: across pieces: {Script: order} (constructor accepts a mapping; + #: stored as sorted pairs). The default reads wholly-Han/Hangul + #: names, and kana-licensed Japanese names, family-first -- see #: :data:`~nameparser.DEFAULT_SCRIPT_ORDERS`. Opt out with #: ``script_orders={}``. Latin-script and mixed-script input is #: never affected. Like name_order, ignored where a comma already diff --git a/tests/v2/cases.py b/tests/v2/cases.py index fb6e50b..e1b2a41 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -522,12 +522,12 @@ def __post_init__(self) -> None: "role = family"), Case("mixed_script_untouched_by_script_orders", "John 王", {"given": "John", "family": "王"}, - notes="single_script is None for a mixed name: script_orders " + notes="effective_script is None for a mixed name: script_orders " "declines and the positional default governs"), Case("two_han_scripts_untouched_by_script_orders", "毛 김", {"given": "毛", "family": "김"}, - notes="two scripts also decline -- ONE script for the whole " - "name"), + notes="two scripts also decline -- the rule is one script, or " + "the Han/kana repertoire the #272 license covers"), Case("zh_unspaced", "毛泽东", {"family": "毛", "given": "泽东"}, diff --git a/tests/v2/pipeline/test_assign.py b/tests/v2/pipeline/test_assign.py index c050890..69544c1 100644 --- a/tests/v2/pipeline/test_assign.py +++ b/tests/v2/pipeline/test_assign.py @@ -194,6 +194,22 @@ def test_script_order_declines_when_no_piece_has_a_script() -> None: assert _by_role(out, Role.FAMILY) == "Smith" +def test_kana_licensed_piece_resolves_via_hiragana_entry() -> None: + # a kanji+hiragana mixed piece set takes the license (#272) and + # resolves through the HIRAGANA table entry, family-first + out = _assigned("高橋 みなみ") + assert _by_role(out, Role.FAMILY) == "高橋" + assert _by_role(out, Role.GIVEN) == "みなみ" + + +def test_pure_katakana_piece_falls_back_to_name_order() -> None: + # katakana alone is not in the table (transcription ambiguity), so + # a pure-katakana piece set falls back to the positional default + out = _assigned("マイケル ジャクソン") + assert _by_role(out, Role.GIVEN) == "マイケル" + assert _by_role(out, Role.FAMILY) == "ジャクソン" + + def test_script_with_no_table_entry_falls_back() -> None: # A single, well-defined script the table simply does not list: # resolution must fall through to name_order rather than pick an diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index ff62311..657c323 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -3,7 +3,7 @@ from nameparser._lexicon import Lexicon from nameparser._pipeline._vocab import ( _SCRIPT_RANGES, effective_script, is_initial, is_suffix_lenient, - is_suffix_strict, single_script, + is_suffix_strict, resolve_script_set, single_script, ) from nameparser._policy import Script @@ -120,6 +120,30 @@ def test_effective_script_kana_license() -> None: assert effective_script("") is None +def test_resolve_script_set_generalizes_the_license_across_pieces() -> None: + # a single script passes through as-is, including a script with no + # order-default entry (KATAKANA): the caller decides what to do + # with that, this function only reports what was found + assert resolve_script_set([Script.HAN]) is Script.HAN + assert resolve_script_set([Script.HAN, Script.HAN]) is Script.HAN + assert resolve_script_set([Script.KATAKANA]) is Script.KATAKANA + # the license, generalized: Han + Hiragana across two SEPARATE, + # individually single-script pieces (never mixed within one + # token) is exactly the repertoire effective_script licenses + # inside a single token, so it collapses to the carrier key + assert resolve_script_set([Script.HAN, Script.HIRAGANA]) \ + is Script.HIRAGANA + assert resolve_script_set([Script.HAN, Script.KATAKANA]) \ + is Script.HIRAGANA + assert resolve_script_set([Script.HIRAGANA, Script.KATAKANA]) \ + is Script.HIRAGANA + # outside the license: Han+Hangul is two real scripts, neither + # kana -- no single tradition, so this declines like the caller's + # own None (Latin, mixed, empty) + assert resolve_script_set([Script.HAN, Script.HANGUL]) is None + assert resolve_script_set([]) is None + + def test_nfd_katakana_still_classifies_and_declines_the_license() -> None: # Built via normalize(), never pasted as decomposed literals -- # NFD "ガガ" is base katakana カ + COMBINING VOICED SOUND MARK diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index 7f50c7a..31a93ac 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -573,6 +573,11 @@ def test_facade_reads_wholly_han_names_family_first() -> None: assert HumanName("毛泽东").first == "" +def test_facade_reads_kana_licensed_names_family_first() -> None: + n = HumanName("高橋 みなみ") + assert (n.last, n.first) == ("高橋", "みなみ") + + def test_facade_parses_unspaced_korean_by_default() -> None: # the default-behavior fix flows through the v1 facade (its # lexicon mirrors Lexicon.default() via the shim snapshot) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 9948019..c5a578f 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -465,6 +465,27 @@ def test_nfd_korean_input_still_reads_family_first() -> None: # the point under test is the ORDER (family first), not encoding assert (unicodedata.normalize("NFC", n.family), unicodedata.normalize("NFC", n.given)) == ("김", "민준") + # pinned, not just stated: a future tokenize-level normalize (the + # tempting over-fix, and an anti-#100 violation) must fail here + assert n.family == unicodedata.normalize("NFD", "김") + + +def test_kana_licensed_names_read_family_first_by_default() -> None: + # the #272 amendment: hiragana identifies Japanese as certainly + # as hangul identifies Korean -- release-log-classified fix + n = parse("高橋 みなみ") + assert (n.family, n.given) == ("高橋", "みなみ") + n = parse("山田 エミ") # kanji piece + katakana piece: native + assert (n.family, n.given) == ("山田", "エミ") + # lone kana-licensed token takes the family role, unsplit + assert parse("高橋みなみ").family == "高橋みなみ" + + +def test_pure_katakana_stays_positional() -> None: + # transcribed foreign names keep their original (usually Western) + # order; katakana alone licenses nothing + n = parse("マイケル ジャクソン") + assert (n.given, n.family) == ("マイケル", "ジャクソン") def test_latin_names_are_untouched_by_script_orders() -> None: @@ -497,9 +518,9 @@ def test_three_cjk_pieces_take_the_script_order_middles() -> None: def test_two_cjk_scripts_fall_back_even_though_both_read_family_first() -> None: - # The rule is ONE script for the whole name, not "the entries - # agree": a Han+Hangul name is written in neither tradition, so it - # takes the positional default. + # The rule is one script, or the Han/kana repertoire the #272 + # license covers -- not "the entries agree": Han+Hangul is written + # in neither tradition, so it takes the positional default. n = parse("毛 김") assert (n.given, n.family) == ("毛", "김") diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 7e1cb8e..3fc6910 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -506,12 +506,20 @@ def test_patched_rejects_non_patch() -> None: def test_script_orders_default_and_canonical_storage() -> None: p = Policy() assert p.script_orders == DEFAULT_SCRIPT_ORDERS + # three entries now (#272 adds HIRAGANA), sorted by Script value: + # "han" < "hangul" < "hiragana" + assert p.script_orders == ( + (Script.HAN, FAMILY_FIRST), + (Script.HANGUL, FAMILY_FIRST), + (Script.HIRAGANA, FAMILY_FIRST), + ) # constructor tolerates a Mapping; storage is the sorted pair # tuple (hashability, the capitalization_exceptions precedent -- # which is also why the mapping spelling needs the ignore: the # field is annotated with what it STORES) q = Policy(script_orders={Script.HANGUL: FAMILY_FIRST, # type: ignore[arg-type] - Script.HAN: FAMILY_FIRST}) + Script.HAN: FAMILY_FIRST, + Script.HIRAGANA: FAMILY_FIRST}) assert q == p and hash(q) == hash(p) assert Policy(script_orders={}).script_orders == () # type: ignore[arg-type] From 811ec5471f8c511ec61923059b50f5b6f8d4ee1c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 02:37:50 -0700 Subject: [PATCH 03/14] Treat the katakana middle dot as a token separator (#272) Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_tokenize.py | 33 +++++++++++++++++++----- nameparser/_pipeline/_vocab.py | 8 +++--- tests/v2/pipeline/test_tokenize.py | 41 ++++++++++++++++++++++++++++++ tests/v2/pipeline/test_vocab.py | 9 ++++--- tests/v2/test_parser.py | 38 +++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 12 deletions(-) diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index ac19b16..28460e4 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -6,10 +6,13 @@ slice), comma_offsets (segmentation points; never tokens). Reads: Policy.strip_emoji, Policy.strip_bidi. -There is NO text-rewriting normalize stage: whitespace collapsing and -emoji/bidi stripping are character-classification rules here -- -ignorable characters act as separators and never enter a token, so -spans always index the original exactly as given. +There is NO text-rewriting normalize stage: whitespace collapsing, +emoji/bidi stripping, and the katakana name-dot split are all +character-classification rules here -- ignorable characters act as +separators and never enter a token, so spans always index the +original exactly as given. Whitespace and the name-dot are +unconditional; emoji/bidi stripping alone is policy-gated +(Policy.strip_emoji/strip_bidi). v1's squash_emoji/squash_bidi REMOVED the char and joined neighbors ('A\U0001f600B' -> 'AB'); here an ignorable char is a SEPARATOR @@ -38,14 +41,32 @@ (0x2600, 0x26FF), (0x2700, 0x27BF)) _BIDI = re.compile('[\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+') +# The katakana middle dot and its halfwidth twin divide the parts of +# a foreign name transcribed into katakana (マイケル・ジャクソン) -- +# native names never contain them, so they separate unconditionally, +# like whitespace (amendment 2026-07-29 section 1b). U+00B7 is +# deliberately NOT here: it is also the Catalan punt volat, interior +# to real names (Gal·la) -- a follow-up issue owns that decision. +_NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65"}) + def _ignorable(ch: str, state: ParseState) -> bool: if ch.isspace(): return True if ch.isascii(): - # both strip classes are entirely non-ASCII (bidi >= U+061C, - # emoji >= U+2600): skip two failing regex calls per letter + # both strip classes and the name-dot separators are entirely + # non-ASCII (bidi >= U+061C, emoji >= U+2600, dots >= U+30FB): + # skip the checks below for every ASCII letter return False + # unconditional, like whitespace -- not policy-gated, so this sits + # ahead of the policy checks rather than in _tokenize_region beside + # COMMA_CHARS (commas RECORD an offset; these dots must not). The + # only load-bearing constraint is "after the isascii fast path" + # (both dots are non-ASCII); being ahead of the bidi/emoji checks + # below is NOT load-bearing -- the three sets are disjoint, so a + # future edit is free to reorder them. + if ch in _NAME_DOT_SEPARATORS: + return True if state.policy.strip_bidi and _BIDI.match(ch): return True if state.policy.strip_emoji: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 27fa505..7054bde 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -66,9 +66,11 @@ # yet every one of them is needed here, and block membership, not the # Script property, is what puts them in range. The katakana block's # upper end (U+30FF) including the middle dot U+30FB is load-bearing -# for effective_script's kana license below (see its docstring) -- a -# later task turns U+30FB into a tokenize-level separator, but until -# then it classifies as ordinary katakana. The ranges below must stay +# for effective_script's kana license below (see its docstring) -- +# tokenize (#272 Task 2b) now turns U+30FB into a token separator +# before this classifier ever sees a split string, but effective_script +# is also called directly on whole strings (e.g. in tests), where the +# dot still classifies as ordinary katakana. The ranges below must stay # mutually disjoint: single_script returns the FIRST covering entry # (dict iteration order), so an overlapping future script would make # the result order-dependent instead of well-defined. diff --git a/tests/v2/pipeline/test_tokenize.py b/tests/v2/pipeline/test_tokenize.py index 4dc4796..96b18c4 100644 --- a/tests/v2/pipeline/test_tokenize.py +++ b/tests/v2/pipeline/test_tokenize.py @@ -89,3 +89,44 @@ def test_emoji_and_bidi_are_ignorable_by_policy() -> None: def test_empty_and_whitespace_yield_no_tokens() -> None: assert _tokenized("").tokens == () assert _tokenized(" ").tokens == () + + +def test_nakaguro_separates_and_enters_no_token() -> None: + # U+30FB divides transcribed foreign names (マイケル・ジャクソン); + # native names never contain it -- amendment 2026-07-29 section 1b + state = _tokenized("マイケル・ジャクソン") + assert [t.text for t in state.tokens] == ["マイケル", "ジャクソン"] + assert all(state.original[t.span.start:t.span.end] == t.text + for t in state.tokens) + # like whitespace, and unlike COMMA_CHARS: it never records an offset + assert state.comma_offsets == () + # leading/trailing dots are plain separators, not content + assert [t.text for t in _tokenized("・ジャクソン").tokens] == ["ジャクソン"] + + +def test_nakaguro_next_to_a_real_comma_still_only_the_comma_offsets() -> None: + # the dot and a real comma side by side must not blur together -- + # the comma still segments and records; the dot still separates + # and stays silent + state = _tokenized("マイケル・, X") + assert [t.text for t in state.tokens] == ["マイケル", "X"] + assert state.comma_offsets == (5,) + + +def test_halfwidth_nakaguro_separates_too() -> None: + # U+FF65 between halfwidth katakana -- build the string from + # escapes and VERIFY the codepoints (the U+F900 homoglyph lesson): + # ﾏｲｹﾙ = halfwidth マイケル (MA I KE RU), + # ･ = the halfwidth dot, ｼﾞｬｸｿﾝ = + # halfwidth ジャクソン (SI + voiced-sound-mark, small-YA, KU, SO, N) + text = ("\uff8f\uff72\uff79\uff99" # \uff8f\uff72\uff79\uff99 = halfwidth MA I KE RU + "\uff65" # \uff65 = halfwidth middle dot + "\uff7c\uff9e\uff6c\uff78\uff7f\uff9d") # halfwidth SI+voice, small-YA, KU, SO, N + # belt and braces: the escapes above must spell the halfwidth dot, + # not the fullwidth U+30FB it could be confused for by eye -- and + # this is exactly the check the un-escaped version was missing: + # it would still pass with U+30FB substituted for U+FF65 + assert "･" in text + state = _tokenized(text) + assert len(state.tokens) == 2 + assert "･" not in "".join(t.text for t in state.tokens) diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index 657c323..d9498c5 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -113,9 +113,12 @@ def test_effective_script_kana_license() -> None: # "マイケル" above -- the license only ever fires on a MIXED # token, and this one isn't mixed. It has no order-default entry # (DEFAULT_SCRIPT_ORDERS carries no KATAKANA key), which is where - # "declines the license" actually shows up. A later task turns - # U+30FB into a tokenize-level separator, so this string arrives - # at effective_script as two tokens instead of one. + # "declines the license" actually shows up. tokenize (#272 Task 2b) + # now turns U+30FB into a token separator, so this exact string + # arrives at effective_script as two tokens during a real parse -- + # this unit test calls effective_script directly on the whole + # string, bypassing tokenize, so it still sees the dot and must + # keep passing unchanged. assert effective_script("マイケル・ジャクソン") is Script.KATAKANA assert effective_script("") is None diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index c5a578f..6e426fd 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -488,6 +488,44 @@ def test_pure_katakana_stays_positional() -> None: assert (n.given, n.family) == ("マイケル", "ジャクソン") +def test_katakana_transcription_parses_by_its_divider() -> None: + # the dot tells us where the parts meet; transcriptions keep the + # source language's order, which the positional default provides + n = parse("マイケル・ジャクソン") + assert (n.given, n.family) == ("マイケル", "ジャクソン") + + +def test_nakaguro_split_han_tokens_take_the_han_order() -> None: + # different code path from the katakana case above: splitting on + # the dot here produces two PURE-HAN tokens (no kana involved), so + # script_orders' family-first HAN entry fires, same as any other + # two-token Han name -- the dot only decides where the split falls + n = parse("高橋・一郎") + assert (n.family, n.given) == ("高橋", "一郎") + + +def test_halfwidth_nakaguro_splits_at_parse_level_too() -> None: + # decision, not accident: halfwidth kana classify as no script at + # all (_SCRIPT_RANGES only covers the fullwidth blocks), so this + # is order-agnostic positional fallback, not a script-order rule -- + # the dot still divides the tokens regardless + text = "ﾏｲｹﾙ･ｼﾞｬｸｿﾝ" + n = parse(text) + assert (n.given, n.family) == ( + "ﾏｲｹﾙ", "ｼﾞｬｸｿﾝ") + + +def test_nakaguro_inside_a_nickname_still_splits() -> None: + # decision, not accident: delimited content tokenizes under the + # same separator rules as the main stream, so a dot inside a + # nickname still divides it -- and re-rendering a token stream + # necessarily uses the render join, so the dot comes back as a + # space, same as any other separator + n = parse("山田 太郎 (マイケル・ジャクソン)") + assert n.nickname == "マイケル ジャクソン" + assert (n.family, n.given) == ("山田", "太郎") + + def test_latin_names_are_untouched_by_script_orders() -> None: n = parse("John Smith") assert (n.given, n.family) == ("John", "Smith") From 9acbef1222a4db52826f185de50464d3416a9900 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 02:58:49 -0700 Subject: [PATCH 04/14] Add the Segmentation protocol and the Parser segmenter hook (#272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pluggable-segmenter seam from the locales spec §4, as amended 2026-07-29: public Segmentation/Segmenter types, a keyword-only Parser(segmenter=...) field, and the value plumbed into ParseState. Nothing consults it yet -- the script_segment stage picks it up in the next commit, pinned here by a strict xfail on the totality exception. Segmentation is an index protocol (strictly ascending interior offsets plus an optional confidence), so a segmenter physically cannot invent, drop, or rewrite characters; Segmentation(()) means "confidently one token" where None declines. parser_for carries base.segmenter through -- packs are pure data and cannot supply one, so base=Parser(segmenter=...) is how a pack and a segmenter combine. The two promises the spec deferred with the hook land with it: Parser pickles iff its segmenter does, and a segmenter's own exceptions propagate rather than being swallowed as content errors. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- nameparser/__init__.py | 4 +- nameparser/_parser.py | 71 ++++++++++++++++++++++++------- nameparser/_pipeline/_state.py | 10 ++++- nameparser/_types.py | 74 ++++++++++++++++++++++++++++++++- tests/v2/pipeline/test_state.py | 1 + tests/v2/test_layering.py | 1 + tests/v2/test_parser.py | 53 ++++++++++++++++++++++- tests/v2/test_reprs.py | 19 +++++++++ tests/v2/test_types.py | 37 ++++++++++++++++- 10 files changed, 250 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6cbb4b9..07e7530 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. -- **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is the optional `Parser(segmenter=...)` hook — a Parser holding a callable pickles iff that callable does, and the same hook is parse-totality's one exception: a user-supplied segmenter's own exceptions propagate rather than being swallowed as content errors). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, and `test_regex_sync.py`, which pins every hand-copied pattern or codepoint table against its source wherever the copy lives — including copies outside the package), names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/nameparser/__init__.py b/nameparser/__init__.py index a433a76..19afbd9 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -27,6 +27,8 @@ AmbiguityKind, ParsedName, Role, + Segmentation, + Segmenter, Span, Token, ) @@ -36,7 +38,7 @@ "HumanName", # v2 core "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", - "STABLE_TAGS", + "STABLE_TAGS", "Segmentation", "Segmenter", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 6389ce6..6aa615f 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -11,7 +11,7 @@ import dataclasses import functools import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from nameparser._lexicon import Lexicon from nameparser._locale import Locale @@ -20,8 +20,8 @@ from nameparser._pipeline._state import ParseState from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch from nameparser._types import ( - FOLDED_TAG, ParsedName, Token, _guarded_getstate, _guarded_setstate, - _validated_field_strings, + FOLDED_TAG, ParsedName, Segmenter, Token, _guarded_getstate, + _guarded_setstate, _validated_field_strings, ) @@ -30,18 +30,37 @@ class Parser: """A configured name parser: a :class:`Lexicon` (vocabulary) plus a :class:`Policy` (behavior), both defaulted when omitted. Build one when you need non-default configuration, build it once, and - call :meth:`parse` many times -- it is immutable, thread-safe, and - picklable by construction: all validity checking happens at + call :meth:`parse` many times -- it is immutable and thread-safe. + + An optional keyword-only ``segmenter`` (a :data:`~nameparser.Segmenter`) + plugs in outside knowledge of where an unspaced CJK token divides -- + Japanese kanji names, which no bundled list can settle. It is + consulted only for a token the segmentation stage gates in and the + vocabulary DECLINES, so a locale pack's surnames always win where + they match; returning None declines in turn and the token stays + whole. Two promises narrow when one is supplied (locales spec §4): + parse-totality gains its one exception -- an exception raised by + the segmenter propagates, because a user-supplied callable's own + error is a user-code error, not a content error -- and this Parser + pickles only if its segmenter does (a module-level function + pickles; a lambda or closure does not). With no segmenter, both + promises hold unconditionally: all validity checking happens at construction, so a Parser that constructs successfully cannot fail at parse time on any str content. (The None field defaults resolve in __post_init__; after - construction both fields are always non-None -- the annotations - state the steady-state truth, hence the assignment ignores on the - defaults.)""" + construction lexicon and policy are always non-None -- the + annotations state the steady-state truth, hence the assignment + ignores on the defaults.)""" lexicon: Lexicon = None # type: ignore[assignment] # None -> default() policy: Policy = None # type: ignore[assignment] # None -> Policy() + #: An optional hook supplying outside knowledge of where an unspaced + #: token divides -- see the class docstring; None leaves such tokens + #: whole. Keyword-only, so the reserved growth stays additive + #: (locales spec §4): positional construction keeps its two-argument + #: shape. + segmenter: Segmenter | None = field(default=None, kw_only=True) # in the class body so @dataclass(slots=True) keeps them __getstate__ = _guarded_getstate @@ -58,16 +77,32 @@ def __post_init__(self) -> None: elif not isinstance(self.policy, Policy): raise TypeError( f"policy must be a Policy or None, got {self.policy!r}") + if self.segmenter is not None and not callable(self.segmenter): + raise TypeError( + f"segmenter must be callable or None, got {self.segmenter!r}") def __repr__(self) -> str: - # composes the two bounded component reprs (spec §2 reprs) - return f"Parser({self.lexicon!r}, {self.policy!r})" + # composes the two bounded component reprs (spec §2 reprs); the + # segmenter shows by name, and only when one is set, so the + # default Parser's repr is unchanged + seg = "" + if self.segmenter is not None: + # never repr() the callable itself: a partial reprs its + # bound arguments and a callable instance its address, both + # unbounded -- the class name is the bounded fallback + name = (getattr(self.segmenter, "__qualname__", None) + or type(self.segmenter).__name__) + seg = f", segmenter={name}" + return f"Parser({self.lexicon!r}, {self.policy!r}{seg})" def parse(self, text: str) -> ParsedName: """Parse one name string into a :class:`ParsedName`. Never raises on string content (unparseable input yields empty fields plus ambiguities); non-str raises TypeError eagerly, - with a decode hint for bytes (bytes support ended with 1.x).""" + with a decode hint for bytes (bytes support ended with 1.x). + The one exception to that totality is a configured + ``segmenter``, whose own exceptions propagate (see the class + docstring).""" if isinstance(text, bytes): raise TypeError( "parse() takes str, not bytes -- decode first, e.g. " @@ -75,7 +110,7 @@ def parse(self, text: str) -> ParsedName: if not isinstance(text, str): raise TypeError(f"parse() takes str, got {text!r}") state = ParseState(original=text, lexicon=self.lexicon, - policy=self.policy) + policy=self.policy, segmenter=self.segmenter) return assemble(run(state)) # -- editing ---------------------------------------------------------- @@ -158,7 +193,10 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: pack are wrapped with that pack's identity (spec §4 amendment) -- PolicyPatch validates lazily, so with stacked packs the raw error would otherwise point at nothing. Two packs setting the same SCALAR - field is a declared conflict: UserWarning, later wins.""" + field is a declared conflict: UserWarning, later wins. base's + ``segmenter`` carries through unchanged -- packs are pure data and + cannot supply one, so ``base=Parser(segmenter=...)`` is how a pack + and a segmenter combine.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") for loc in locales: @@ -166,6 +204,11 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") lexicon = base.lexicon if base is not None else Lexicon.default() policy = base.policy if base is not None else Policy() + # Unpacked here with its siblings because the return builds a FRESH + # Parser: any field not listed there silently takes its default, and + # a dropped segmenter would be invisible (packs are pure data and + # cannot supply one, so base is its only source). + segmenter = base.segmenter if base is not None else None scalar_setters: dict[str, str] = {} for loc in locales: for f in dataclasses.fields(PolicyPatch): @@ -189,4 +232,4 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: # a subclass with extra mandatory args would break this rewrap raise type(exc)( f"while applying locale {loc.code!r}: {exc}") from exc - return Parser(lexicon=lexicon, policy=policy) + return Parser(lexicon=lexicon, policy=policy, segmenter=segmenter) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index dcba311..ba06c5b 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -15,7 +15,7 @@ from nameparser._lexicon import Lexicon from nameparser._policy import Policy -from nameparser._types import AmbiguityKind, Role, Span +from nameparser._types import AmbiguityKind, Role, Segmenter, Span # The comma characters (ASCII/Arabic/fullwidth, #265). Shared here so @@ -79,11 +79,17 @@ class ParseState: different stages needs an emitter in each. Post-group, segments may retain indices of dropped tokens -- assign iterates pieces, never segments. This ownership map is pinned by - tests/v2/pipeline/test_state.py.""" + tests/v2/pipeline/test_state.py. + + segmenter belongs to no stage: like original/lexicon/policy it is + passed in at construction by Parser.parse and only ever READ (by + script_segment, for a token the vocabulary declined).""" original: str lexicon: Lexicon policy: Policy + #: The optional Parser(segmenter=...) hook; None = not configured. + segmenter: Segmenter | None = None extracted: tuple[tuple[Role, Span], ...] = () masked: tuple[Span, ...] = () tokens: tuple[WorkToken, ...] = () diff --git a/nameparser/_types.py b/nameparser/_types.py index a33121d..e49fe0a 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -14,7 +14,7 @@ from __future__ import annotations import dataclasses -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass from enum import Enum, StrEnum from typing import TYPE_CHECKING, NamedTuple, NoReturn, TypeVar @@ -230,6 +230,78 @@ def __repr__(self) -> str: return f"Token({self.text!r} {where} {self.role.name}{tags})" +@dataclass(frozen=True, slots=True) +class Segmentation: + """A segmenter's answer for one unspaced token: the interior offsets + to split at (each offset begins a new piece, so + ``Segmentation((2,))`` cuts a three-character token into + ``token[:2]`` and ``token[2:]``; strictly ascending, each >= 1 -- an + index protocol, so a segmenter physically cannot invent, drop, or + rewrite characters) and an optional confidence in [0, 1]. + ``Segmentation(())`` means "confidently one token" -- distinct from + returning None, which DECLINES ("I don't know"). The upper bound + (< len(token)) is validated by the consuming stage, which knows the + text.""" + + #: Interior character offsets to split at, ascending. + splits: tuple[int, ...] + #: How sure the segmenter is, or None for "no opinion". + confidence: float | None = None + + # in the class body so @dataclass(slots=True) keeps them + __getstate__ = _guarded_getstate + __setstate__ = _guarded_setstate + + def __post_init__(self) -> None: + # The same guard Token.tags carries: a mapping would silently + # contribute only its keys, and a bare int would surface as an + # uncurated "not iterable" from the tuple() below. + if isinstance(self.splits, Mapping): + raise TypeError( + "Segmentation.splits must be an iterable of integers, " + "not a mapping" + ) + try: + splits = tuple(self.splits) + except TypeError: + raise TypeError( + f"Segmentation.splits must be an iterable of integers, " + f"got {self.splits!r}" + ) from None + for offset in splits: + # bool is an int subclass: True as an offset is a comparison + # result leaking into an index slot, not a split point + if isinstance(offset, bool) or not isinstance(offset, int): + raise TypeError( + f"Segmentation.splits must be integers, got {offset!r}") + if offset < 1: + raise ValueError( + f"Segmentation.splits must be interior offsets " + f"(each >= 1), got {offset}") + if any(b <= a for a, b in zip(splits, splits[1:])): + raise ValueError( + f"Segmentation.splits must be strictly ascending, " + f"got {splits!r}") + object.__setattr__(self, "splits", splits) + conf = self.confidence + if conf is not None: + if isinstance(conf, bool) or not isinstance(conf, (int, float)): + raise TypeError( + f"Segmentation.confidence must be a float or None, " + f"got {conf!r}") + # stored as given, not coerced to float: an int 1 is a valid + # confidence and the range check is what the callers rely on + if not 0.0 <= conf <= 1.0: + raise ValueError( + f"Segmentation.confidence must be within [0, 1], " + f"got {conf!r}") + + +#: The segmenter hook's shape: token text in, :class:`Segmentation` out, +#: None to decline. Plug one in via ``Parser(segmenter=...)``. +Segmenter = Callable[[str], Segmentation | None] + + class AmbiguityKind(StrEnum): """The stable vocabulary of :class:`Ambiguity` kinds. A StrEnum: members ARE their string values, so ``kind == "particle-or-given"`` diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index a02908d..f927ec1 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -16,6 +16,7 @@ def test_state_defaults_are_empty() -> None: assert s.structure is Structure.NO_COMMA assert s.ambiguities == () and s.extracted == () and s.masked == () assert s.comma_offsets == () and s.dropped == () and s.piece_tags == () + assert s.segmenter is None def test_state_is_frozen_and_replace_works() -> None: diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 8739bf1..081b7e7 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -167,6 +167,7 @@ def test_lexicon_never_imports_config_package_root_or_parser() -> None: def test_public_exports() -> None: expected = { "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "Segmentation", "Segmenter", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "Script", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "DEFAULT_SCRIPT_ORDERS", "Locale", diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 6e426fd..cca4176 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -4,11 +4,13 @@ import pytest -from nameparser import Lexicon, Locale, Parser, Policy, PolicyPatch, parse, parser_for +from nameparser import ( + Lexicon, Locale, Parser, Policy, PolicyPatch, locales, parse, parser_for, +) from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, ) -from nameparser._types import AmbiguityKind, Role +from nameparser._types import AmbiguityKind, Role, Segmentation def test_parser_defaults_and_properties() -> None: @@ -593,3 +595,50 @@ def test_unspaced_korean_names_parse_by_default() -> None: assert (n.family, n.given) == ("김", "민준") n = parse("남궁민수") # two-syllable surname beats single 남 assert (n.family, n.given) == ("남궁", "민수") + + +# -- the segmenter hook (#272) ------------------------------------------ + + +def _module_level_decline(text: str) -> Segmentation | None: + return None # module-level so pickle can find it + + +def test_parser_segmenter_is_keyword_only_and_validated() -> None: + assert Parser().segmenter is None + with pytest.raises(TypeError, match="callable"): + Parser(segmenter=5) # type: ignore[arg-type] + with pytest.raises(TypeError, match="positional"): + Parser(Lexicon.default(), Policy(), None) # type: ignore[misc] # positional: rejected + + +def test_parser_for_carries_the_base_segmenter() -> None: + def seg(text: str) -> Segmentation | None: + return None + p = parser_for(locales.get("zh"), base=Parser(segmenter=seg)) + assert p.segmenter is seg + + +def test_parser_picklability_is_conditional_on_the_segmenter() -> None: + # declared contract (locales spec section 4): Parser pickles iff + # its segmenter does -- like any callable-holding object + p = pickle.loads(pickle.dumps(Parser(segmenter=_module_level_decline))) + assert p.segmenter is _module_level_decline + unpicklable = Parser(segmenter=lambda t: None) # constructs fine + with pytest.raises(Exception): # pickle's exception type varies + pickle.dumps(unpicklable) # only pickling fails + + +@pytest.mark.xfail(strict=True, + reason="the stage consults the segmenter in the next commit") +def test_segmenter_exceptions_propagate() -> None: + # the ONE exception to parse-totality (locales spec section 4, + # declared 2026-07-11): a user-supplied callable's own error is a + # user-code error, not a content error, and must not be swallowed + def boom(text: str) -> Segmentation | None: + raise RuntimeError("segmenter bug") + + p = parser_for(locales.get("zh"), base=Parser(segmenter=boom)) + with pytest.raises(RuntimeError, match="segmenter bug"): + p.parse("阿明") # zh pack active, 阿 unmatched by vocabulary -> + # the segmenter will be consulted once Task 4 lands diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 71ede1f..3940f83 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -1,4 +1,5 @@ import dataclasses +import functools from typing import cast import pytest @@ -10,6 +11,10 @@ from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token +def _decline(text: str, blob: str = "") -> None: + return None # a stand-in segmenter for the Parser repr cases + + def test_token_repr_is_compact() -> None: t = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert repr(t) == "Token('de' @9:11 FAMILY {particle})" @@ -91,6 +96,7 @@ def test_policy_patch_repr_shows_only_set_fields() -> None: Lexicon.default(), Lexicon.empty(), Parser(), + Parser(segmenter=functools.partial(_decline, blob="x" * 500)), ]) def test_config_reprs_never_leak_the_unset_sentinel(obj: object) -> None: # Guard the whole family: every config type's repr is bounded and @@ -99,6 +105,19 @@ def test_config_reprs_never_leak_the_unset_sentinel(obj: object) -> None: assert "_Unset" not in repr(obj) +def test_parser_repr_names_the_segmenter_without_its_contents() -> None: + # A callable is the one component whose own repr is unbounded: a + # partial spells out its bound arguments, a callable instance its + # memory address. Neither may reach Parser's repr, so the name comes + # from __qualname__ when there is one and the TYPE otherwise. + plain = repr(Parser(segmenter=_decline)) + assert "segmenter=_decline" in plain + partial = repr(Parser(segmenter=functools.partial(_decline, + blob="x" * 500))) + assert "segmenter=partial" in partial + assert "xxx" not in partial and len(partial) < len(plain) + 20 + + def test_policy_patch_repr_survives_unvalidated_name_order() -> None: # PolicyPatch defers name_order validation to apply time, so a # patch can hold a non-canonical or even garbage tuple -- its repr diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 8390fc5..6904db4 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,10 +1,12 @@ +import dataclasses from collections.abc import Iterable import pytest from nameparser import parse from nameparser._types import ( - STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token, + STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, + Span, Token, ) @@ -73,6 +75,37 @@ def test_token_is_frozen_and_hashable() -> None: assert hash(t) == hash(Token("Juan", Span(0, 4), Role.GIVEN)) +def test_segmentation_validates_splits() -> None: + s = Segmentation((2,), confidence=0.97) + assert s.splits == (2,) and s.confidence == 0.97 + assert Segmentation(()).confidence is None + with pytest.raises(ValueError, match="ascending"): + Segmentation((3, 2)) + with pytest.raises(ValueError, match="interior"): + Segmentation((0,)) + with pytest.raises(TypeError, match="integers"): + Segmentation(("2",)) # type: ignore[arg-type] + with pytest.raises(ValueError, match="confidence"): + Segmentation((2,), confidence=1.5) + + +def test_segmentation_rejects_mappings_and_non_iterables() -> None: + # Token.tags' guards, on the other collection field: a mapping + # would silently contribute only its keys, and a bare int would + # surface as an uncurated "not iterable" naming nothing + with pytest.raises(TypeError, match="Segmentation.splits"): + Segmentation({2: "a", 3: "b"}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="Segmentation.splits"): + Segmentation(5) # type: ignore[arg-type] + + +def test_segmentation_is_frozen_and_hashable() -> None: + s = Segmentation((2,)) + assert isinstance(hash(s), int) + with pytest.raises(dataclasses.FrozenInstanceError): + s.splits = (3,) # type: ignore[misc] + + def test_ambiguity_kind_members_are_their_string_values() -> None: assert AmbiguityKind.PARTICLE_OR_GIVEN == "particle-or-given" assert AmbiguityKind("order") is AmbiguityKind.ORDER @@ -452,3 +485,5 @@ def test_with_field_tokens_rejects_mismatched_roles() -> None: with pytest.raises(ValueError, match="has role family, not given"): name._with_field_tokens( {Role.GIVEN: (Token("x", None, Role.FAMILY),)}) + + From 51a259af142f1830525bd87233e088a134de39d1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 03:26:22 -0700 Subject: [PATCH 05/14] Consult the segmenter on vocabulary decline, n-way splits (#272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script_segment stage now reaches the Parser(segmenter=...) hook it has been carrying since Task 3. Vocabulary first, segmenter on decline, so parser_for(ZH, JA, segmenter=...) composes: a listed surname is a dictionary certainty and wins, and the segmenter takes what is left. Its Segmentation may cut any number of times, so the split path is written for n cuts and the vocabulary hit is its one-cut case -- one implementation, so the two sources cannot drift apart on index arithmetic. The token gate swaps single_script for effective_script: kana-licensed composites (高橋みなみ) resolve to the HIRAGANA carrier and gate in under the JA pack's set, while pure katakana resolves to KATAKANA, which no activation set contains. A declined answer (None), a confident whole (empty splits), and an out-of-bounds one (a buggy segmenter -- Segmentation never sees the text, so the upper bound is the stage's to check) all leave the token alone. The segmenter's own exceptions propagate, the declared totality exception, which flips Task 3's strict xfail to a live test. An answer scoring under _SEGMENTER_CONFIDENCE_FLOOR attaches a SEGMENTATION report naming the score; the floor is a placeholder pending Task 5's measurement against namedivider's real distribution. Also narrows Segmentation's non-iterable guard to probe iter() rather than wrapping tuple(): a TypeError raised inside a caller's generator is that generator's bug and was being relabeled as "splits must be an iterable". Co-Authored-By: Claude Fable 5 --- nameparser/_pipeline/_script_segment.py | 226 ++++++++++++++++------- nameparser/_pipeline/_state.py | 6 +- nameparser/_types.py | 7 +- tests/v2/pipeline/test_script_segment.py | 190 ++++++++++++++++++- tests/v2/pipeline/test_state.py | 7 +- tests/v2/test_parser.py | 27 ++- tests/v2/test_reprs.py | 11 +- tests/v2/test_types.py | 19 +- 8 files changed, 415 insertions(+), 78 deletions(-) diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 992226f..e9cbe09 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -1,29 +1,44 @@ -"""Stage: script_segment (#271). +"""Stage: script_segment (#271, #272). -Consumes: tokens, segments, structure. +Consumes: tokens, segments, structure, segmenter. Produces: tokens (the first activated-script token of the name -segment split in two), segments (index runs remapped past the -insertion), ambiguities (indices likewise remapped, plus a -SEGMENTATION report when more than one split was -vocabulary-supported). -Reads: Policy.segment_scripts, Lexicon.surnames. +segment split into n+1 sub-slices), segments (index runs remapped past +the insertions), ambiguities (indices likewise remapped, plus a +SEGMENTATION report when more than one split was vocabulary-supported, +or when a segmenter's answer scored under the confidence floor). +Reads: Policy.segment_scripts, Lexicon.surnames, ParseState.segmenter. Unspaced CJK names give tokenize no separator to find, so this stage inserts the missing token boundary by vocabulary: the first token -written wholly in an activated script is matched longest-first against +written in an activated script is matched longest-first against Lexicon.surnames, and a hit splits it in two. Compound-before-single ("夏侯惇" is 夏侯 + 惇, though 夏 is itself a surname) falls out of -longest-first. The split makes two sub-slices of the one token, -rewriting nothing -- spans still index the original exactly, so the -anti-#100 invariant holds by construction. +longest-first. The split makes sub-slices of the one token, rewriting +nothing -- spans still index the original exactly, so the anti-#100 +invariant holds by construction. + +Where the VOCABULARY declines -- no prefix matched -- an optional +Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). +Vocabulary first, segmenter on decline, so parser_for(ZH, JA, +segmenter=...) composes: a listed surname is a dictionary certainty +and wins, and the segmenter takes what is left. Its Segmentation may +cut anywhere and any number of times, which is why the split path +below is written for n cuts and the vocabulary hit is simply its +one-cut case. A segmenter's own exceptions PROPAGATE -- the single +declared exception to parse totality (locales spec section 4): a +user-supplied callable's error is a user-code error, not a content +error. Placed AFTER segment, on the comma doctrine that script-conditional behavior is ignored where a comma already decides the family (the rule script_orders follows): under FAMILY_COMMA the pre-comma text IS the family by declaration, and splitting it would invent a boundary the writer explicitly did not draw -- "남궁민수, 지훈" must render -family "남궁민수", not "남궁 민수". The post-comma side is given-name -text, no surname site either, so that structure opts out whole. +family "남궁민수", not "남궁 민수". That doctrine is about the input's +structure, not the split's source, so it covers the segmenter +identically: the opt-out runs before either is consulted. The +post-comma side is given-name text, no surname site either, so that +structure opts out whole. NO_COMMA and SUFFIX_COMMA still split, within segments[0] (the name part) only. Running after segment costs the index remaps below; running BEFORE it would have made the comma structure itself depend @@ -39,6 +54,11 @@ default-shipped), while HAN is opt-in via locales.ZH -- a Chinese surname list corrupts Japanese names ("高橋一郎" must not split as 高 + 橋一郎), which is #272's pluggable segmenter, not this table's. +The gate resolves each token through effective_script, the same +function order resolution uses, so kana-licensed composites (高橋みなみ +-> HIRAGANA) gate in under the JA pack's HIRAGANA entry while +pure-katakana tokens (-> KATAKANA, in no activation set by design -- +they are predominantly transcribed foreign names) never do. Only the FIRST activated-script token is considered, match or no match: family-first traditions put the surname at the front of the name, and a match deeper in the token stream would be a given name or @@ -50,25 +70,86 @@ import functools from nameparser._pipeline._state import ( - ParseState, PendingAmbiguity, Structure, + ParseState, PendingAmbiguity, Structure, WorkToken, ) -from nameparser._pipeline._vocab import single_script -from nameparser._types import AmbiguityKind, Span +from nameparser._pipeline._vocab import effective_script +from nameparser._types import AmbiguityKind, Segmentation, Span + +#: Segmenter answers scoring below this attach a SEGMENTATION report +#: (amendment 2026-07-29 section 3). Placeholder pending the Task-5 +#: measurement against namedivider's real score distribution -- revisit +#: there, not here. Not configurable in 2.x (YAGNI). +_SEGMENTER_CONFIDENCE_FLOOR = 0.9 -def _remap(run: tuple[int, ...], split_at: int) -> tuple[int, ...]: - """One segment run after token `split_at` split in two: the second - half joins the run immediately after it, and every later index - shifts.""" +def _remap(run: tuple[int, ...], split_at: int, + added: int) -> tuple[int, ...]: + """One segment run after token `split_at` split into `added` + 1 + pieces: the extra pieces join the run immediately after it, and + every later index shifts by as many.""" out: list[int] = [] for j in run: if j == split_at: - out.extend((split_at, split_at + 1)) + out.extend(range(split_at, split_at + added + 1)) else: - out.append(j + 1 if j > split_at else j) + out.append(j + added if j > split_at else j) return tuple(out) +def _pieces(text: str, splits: tuple[int, ...]) -> tuple[str, ...]: + """`text` cut at every offset in `splits`: n offsets, n+1 pieces. + One idiom for both callers -- the split itself and the report that + describes it must never disagree about what the cuts produce.""" + cuts = (0, *splits, len(text)) + return tuple(text[a:b] for a, b in zip(cuts, cuts[1:])) + + +def _split(state: ParseState, i: int, splits: tuple[int, ...], + detail: str | None) -> ParseState: + """Cut token `i` at every offset in `splits`, recording `detail` as + a SEGMENTATION report when there is one. + + The offsets arrive non-empty, ascending and interior whatever chose + them. From a segmenter: non-empty is the caller's own check, and + strictly ascending with each >= 1 is Segmentation.__post_init__'s, + while the last offset being < len(text) is the caller's bounds + check -- that half cannot live in Segmentation, which never sees + the text. From the vocabulary: the single offset is >= 1 and + < len(text) by the range(cap, 0, -1) construction and its len-1 + cap. + + The ONE split path: the vocabulary hit is the single-offset case + and the segmenter's answer the general one, so neither can drift + from the other's index arithmetic.""" + token = state.tokens[i] + base = token.span.start + parts: list[WorkToken] = [] + start = 0 + for piece in _pieces(token.text, splits): + end = start + len(piece) + parts.append(dataclasses.replace( + token, text=piece, span=Span(base + start, base + end))) + start = end + added = len(splits) + tokens = state.tokens[:i] + tuple(parts) + state.tokens[i + 1:] + # Every index the earlier stages recorded is now stale past the + # split point: the segment runs group's own iteration rests on, and + # the ambiguities from extract_delimited (resolved to indices by + # tokenize) and segment. An ambiguity ON the split token keeps + # pointing at the head. + segments = tuple(_remap(run, i, added) for run in state.segments) + ambiguities = tuple( + dataclasses.replace(a, indices=tuple( + j + added if j > i else j for j in a.indices)) + for a in state.ambiguities) + if detail is not None: + ambiguities += (PendingAmbiguity( + AmbiguityKind.SEGMENTATION, detail, + tuple(range(i, i + added + 1))),) + return dataclasses.replace(state, tokens=tokens, segments=segments, + ambiguities=ambiguities) + + @functools.lru_cache(maxsize=8) def _longest_entry(surnames: frozenset[str]) -> int: """The longest surname in a vocabulary, cached per-vocabulary @@ -80,8 +161,8 @@ def _longest_entry(surnames: frozenset[str]) -> int: without ever evicting in normal use. Callers must pass a NON-EMPTY vocabulary: max() of an empty set - raises, and the stage's own emptiness guard (`not surnames`) runs - first, so the only call site cannot reach it.""" + raises, and the only call site sits under the stage's `if surnames` + match guard, so it cannot reach that.""" return max(map(len, surnames)) @@ -92,8 +173,8 @@ def script_segment(state: ParseState) -> ParseState: # in any script's ranges return state scripts = state.policy.segment_scripts - surnames = state.lexicon.surnames - if not scripts or not surnames or not state.segments: + # an empty VOCABULARY deliberately does not bail here -- see below + if not scripts or not state.segments: return state if state.structure is Structure.FAMILY_COMMA: return state # the comma already drew the boundary @@ -103,11 +184,12 @@ def script_segment(state: ParseState) -> ParseState: # extracted nickname/maiden content is unreachable from here -- # no input can produce it, so no test pins it. i = next((i for i in state.segments[0] - if single_script(state.tokens[i].text) in scripts), None) + if effective_script(state.tokens[i].text) in scripts), None) if i is None: return state token = state.tokens[i] text = token.text + surnames = state.lexicon.surnames # A token that IS a surname never splits: a bare "남궁" must not # become 남 + 궁 just because the single-syllable surname also # matches -- there is nothing to split off, and a lone token's @@ -117,39 +199,59 @@ def script_segment(state: ParseState) -> ParseState: # Longest-first (compound-before-single falls out of it), capped # so the remainder is never empty. Direct membership, no # _normalize: the script gate admits only CJK text, which the - # storage fold stores unchanged. - cap = min(_longest_entry(surnames), len(text) - 1) - matches = [length for length in range(cap, 0, -1) - if text[:length] in surnames] - if not matches: + # storage fold stores unchanged. An empty vocabulary skips the + # match rather than bailing the stage (_longest_entry's max() has + # nothing to take): a surname-less lexicon declines every token, + # which is exactly the condition the segmenter is consulted on, so + # an early bail would make a configured segmenter silently inert + # under Lexicon.empty() -- the JA pack's own shape. + matches: list[int] = [] + if surnames: + cap = min(_longest_entry(surnames), len(text) - 1) + matches = [length for length in range(cap, 0, -1) + if text[:length] in surnames] + if matches: + take = matches[0] + detail = None + if len(matches) > 1: + # more than one vocabulary-supported split: longest-first + # DECIDED a fork, and the deciding stage records it. A + # single-match split chose nothing, so it stays silent -- + # a dictionary certainty and the statistical guess below + # are different epistemic states, and these two emission + # rules say so. + chosen = " + ".join(map(repr, _pieces(text, (take,)))) + other = " + ".join(map(repr, _pieces(text, (matches[1],)))) + detail = (f"{text!r} splits as {chosen} on the longest " + f"surname; {other} also reads") + return _split(state, i, (take,), detail) + if state.segmenter is None: return state # the first activated-script token decides - take = matches[0] - cut = token.span.start + take - head = dataclasses.replace(token, text=text[:take], - span=Span(token.span.start, cut)) - tail = dataclasses.replace(token, text=text[take:], - span=Span(cut, token.span.end)) - tokens = state.tokens[:i] + (head, tail) + state.tokens[i + 1:] - # Every index the earlier stages recorded is now stale past the - # split point: the segment runs group's own iteration rests on, - # and the ambiguities from extract_delimited (resolved to indices - # by tokenize) and segment. An ambiguity ON the split token keeps - # pointing at the head. - segments = tuple(_remap(run, i) for run in state.segments) - ambiguities = tuple( - dataclasses.replace(a, indices=tuple( - j + 1 if j > i else j for j in a.indices)) - for a in state.ambiguities) - if len(matches) > 1: - # more than one vocabulary-supported split: longest-first - # DECIDED a fork, and the deciding stage records it. A - # single-match split chose nothing, so it stays silent. - alt = matches[1] - ambiguities += (PendingAmbiguity( - AmbiguityKind.SEGMENTATION, - f"{text!r} splits as {text[:take]!r} + {text[take:]!r} " - f"on the longest surname; {text[:alt]!r} + " - f"{text[alt:]!r} also reads", - (i, i + 1)),) - return dataclasses.replace(state, tokens=tokens, segments=segments, - ambiguities=ambiguities) + # No try/except around the call: the module docstring's totality + # exception. The check below is that same doctrine, curated -- a + # duck-typed answer carrying a .splits of its own would otherwise + # wander into the split path and surface as a ValueError naming + # Token, pointing the reader at nameparser's insides instead of at + # their segmenter. Bounded like every message here: the type's + # NAME, never its contents. + answer = state.segmenter(text) + if answer is not None and not isinstance(answer, Segmentation): + raise TypeError( + f"segmenter must return Segmentation or None, got " + f"{type(answer).__name__}") + if answer is None or not answer.splits: + return state # declined, or confidently one token + # splits[-1] is the max -- Segmentation enforced ascending + if answer.splits[-1] >= len(text): + # the upper bound is ours to check (Segmentation never sees the + # text, so it cannot); a violating answer is a buggy segmenter, + # treated as a decline -- never a crash + return state + conf = answer.confidence + detail = None + if conf is not None and conf < _SEGMENTER_CONFIDENCE_FLOOR: + reading = " + ".join(map(repr, _pieces(text, answer.splits))) + detail = (f"{text!r} splits as {reading} on a segmenter answer " + f"scoring {conf:.2f}, under the " + f"{_SEGMENTER_CONFIDENCE_FLOOR} confidence floor") + return _split(state, i, answer.splits, detail) diff --git a/nameparser/_pipeline/_state.py b/nameparser/_pipeline/_state.py index ba06c5b..6cf05bd 100644 --- a/nameparser/_pipeline/_state.py +++ b/nameparser/_pipeline/_state.py @@ -68,9 +68,9 @@ class ParseState: extract_delimited -> extracted/masked; tokenize -> tokens (span- sorted)/comma_offsets; segment -> segments/structure; script_segment -> tokens and segments again (the one stage that - changes the token COUNT: an unspaced CJK token splits in two, - still as sub-slices of the original, and every later index in the - segment runs shifts); classify -> token tags; group -> + changes the token COUNT: an unspaced CJK token splits into n+1 + pieces, still as sub-slices of the original, and every later index + in the segment runs shifts by n); classify -> token tags; group -> pieces/piece_tags/dropped AND maiden token roles; assign/post_rules -> the remaining token roles. Ambiguities are recorded by every stage that DECIDES one -- extract (resolved to a diff --git a/nameparser/_types.py b/nameparser/_types.py index e49fe0a..cccf42e 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -262,12 +262,17 @@ def __post_init__(self) -> None: "not a mapping" ) try: - splits = tuple(self.splits) + iter(self.splits) except TypeError: raise TypeError( f"Segmentation.splits must be an iterable of integers, " f"got {self.splits!r}" ) from None + # OUTSIDE the try on purpose: iter() cannot run a generator's + # body, but tuple() can, and a TypeError raised in there is the + # caller's own bug -- relabeling it "must be an iterable" would + # send the reader to the wrong place entirely. + splits = tuple(self.splits) for offset in splits: # bool is an int subclass: True as an offset is a comparison # result leaking into an index slot, not a split point diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 22ad2d6..d933d1e 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -1,6 +1,9 @@ -"""Stage: script_segment (#271) -- unspaced CJK surname splitting.""" +"""Stage: script_segment (#271, #272) -- unspaced CJK splitting, by +vocabulary and by the pluggable segmenter behind it.""" import dataclasses +import pytest + from nameparser import Parser from nameparser._lexicon import Lexicon from nameparser._pipeline._script_segment import script_segment @@ -10,10 +13,14 @@ ) from nameparser._pipeline._tokenize import tokenize from nameparser._policy import Policy, Script -from nameparser._types import AmbiguityKind +from nameparser._types import AmbiguityKind, Segmentation, Segmenter _HAN = Policy(segment_scripts=frozenset({Script.HAN})) _HANGUL = Policy() # HANGUL is the default activation +# the JA pack's own activation (amendment 2026-07-29): HIRAGANA is the +# kana license's carrier key, so a kanji+kana composite gates in +# through it; KATAKANA is deliberately absent from every set. +_JA = Policy(segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})) # 欧 AND 欧阳 both present on purpose: compound-before-single must # pick 欧阳 (the issue's own acceptance example); same for 남/남궁. # "jr" so the comma cases can reach SUFFIX_COMMA. @@ -21,9 +28,10 @@ suffix_words=frozenset({"jr"})) -def _run(text: str, policy: Policy = _HAN, - lexicon: Lexicon = _LEX) -> ParseState: - state = ParseState(original=text, lexicon=lexicon, policy=policy) +def _run(text: str, policy: Policy = _HAN, lexicon: Lexicon = _LEX, + segmenter: Segmenter | None = None) -> ParseState: + state = ParseState(original=text, lexicon=lexicon, policy=policy, + segmenter=segmenter) return script_segment(segment(tokenize(state))) @@ -31,6 +39,20 @@ def _texts(state: ParseState) -> list[str]: return [t.text for t in state.tokens] +def _fake(splits: tuple[int, ...], + confidence: float | None = None) -> Segmenter: + """A segmenter that gives the same answer whatever it is handed: + every case below pins what the STAGE does with an answer, not how a + real segmenter arrives at one (no external dependency here).""" + def segmenter(text: str) -> Segmentation | None: + return Segmentation(splits, confidence=confidence) + return segmenter + + +def _declines(text: str) -> Segmentation | None: + return None + + def test_splits_leading_surname_and_spans_index_the_original() -> None: state = _run("毛泽东") assert _texts(state) == ["毛", "泽东"] @@ -187,3 +209,161 @@ def test_delimited_only_input_reaches_the_empty_segments_guard() -> None: # at all -- the guard the stage-level helper above cannot reach nick = Parser(lexicon=_LEX, policy=_HANGUL).parse("(민준)") assert nick.nickname == "민준" + + +# -- the pluggable segmenter, consulted on decline (#272) --------------- + + +def test_segmenter_splits_on_vocabulary_decline() -> None: + # the baseline: with no segmenter configured a declined token is + # simply left whole, so every split below is the hook's doing + assert _texts(_run("山田太郎", policy=_JA)) == ["山田太郎"] + # 山田太郎 has no vocabulary surname prefix at all, so the + # vocabulary declines and the token reaches the segmenter + state = _run("山田太郎", policy=_JA, segmenter=_fake((2,))) + assert _texts(state) == ["山田", "太郎"] + assert [(t.span.start, t.span.end) for t in state.tokens] == [ + (0, 2), (2, 4)] + assert all(state.original[t.span.start:t.span.end] == t.text + for t in state.tokens) + + +def test_vocabulary_wins_over_the_segmenter() -> None: + # precedence, and the reason parser_for(ZH, JA, segmenter=...) + # composes: a matched surname is a dictionary certainty, so the + # segmenter is not even asked + asked: list[str] = [] + + def seg(text: str) -> Segmentation | None: + asked.append(text) + return Segmentation((2,)) + + state = _run("毛泽东", policy=_JA, segmenter=seg) + assert _texts(state) == ["毛", "泽东"] + assert asked == [] + + +def test_segmenter_decline_and_confident_whole_are_distinct() -> None: + # two different ANSWERS -- None is "I don't know", () is + # "confidently one token" -- with the same effect on the tokens: + # the difference is the protocol's to carry, not this stage's + for seg in (_declines, _fake(())): + assert _texts(_run("山田太郎", policy=_JA, segmenter=seg)) == \ + ["山田太郎"] + + +def test_kana_licensed_token_gates_in_for_the_segmenter() -> None: + # the gate reads effective_script, not single_script: 高橋みなみ is + # mixed Han+hiragana, which the kana license resolves to the + # HIRAGANA carrier entry -- and HIRAGANA is in the JA activation + state = _run("高橋みなみ", policy=_JA, segmenter=_fake((2,))) + assert _texts(state) == ["高橋", "みなみ"] + + +def test_pure_katakana_never_gates_in() -> None: + # the license's other half: a lone katakana token is predominantly + # a transcribed foreign name, so KATAKANA sits in no activation set + # and no segmenter is consulted for one + assert _texts(_run("マイケル", policy=_JA, segmenter=_fake((2,)))) \ + == ["マイケル"] + + +def test_kana_licensed_token_stays_whole_under_default_policy() -> None: + # the pack IS the language declaration: without it the default + # activation is HANGUL alone, so nothing gates in and the + # configured segmenter is inert + assert _texts(_run("高橋みなみ", segmenter=_fake((2,)))) == \ + ["高橋みなみ"] + + +def test_segmenter_multi_split_produces_n_plus_one_tokens() -> None: + # n cuts make n+1 sub-slices, and every index the earlier stages + # recorded shifts by n rather than by one + state = segment(tokenize(ParseState( + original="山田太 X", lexicon=_LEX, policy=_JA, + segmenter=_fake((1, 2))))) + state = dataclasses.replace(state, ambiguities=( + PendingAmbiguity(AmbiguityKind.UNBALANCED_DELIMITER, + "synthetic", indices=(1,)),)) + out = script_segment(state) + assert _texts(out) == ["山", "田", "太", "X"] + assert [(t.span.start, t.span.end) for t in out.tokens] == [ + (0, 1), (1, 2), (2, 3), (4, 5)] + assert all(out.original[t.span.start:t.span.end] == t.text + for t in out.tokens) + assert out.segments == ((0, 1, 2, 3),) + assert out.ambiguities[0].indices == (3,) + + +def test_low_confidence_emits_segmentation_ambiguity() -> None: + # a statistical guess is a fork the stage decided, so a score under + # _SEGMENTER_CONFIDENCE_FLOOR is reported -- unlike a vocabulary + # split, which reports only when a SECOND surname also matched + state = _run("山田太郎", policy=_JA, + segmenter=_fake((2,), confidence=0.42)) + assert [a.kind for a in state.ambiguities] == [ + AmbiguityKind.SEGMENTATION] + assert "0.42" in state.ambiguities[0].detail + assert state.ambiguities[0].indices == (0, 1) + + +def test_high_confidence_and_no_confidence_emit_nothing() -> None: + # a confident answer is not a fork, and an opinionless one gives + # the stage nothing to judge -- neither is worth a report + # 0.9 is the floor itself: the comparison is strict, so an answer + # AT the floor is not under it + for conf in (0.99, 0.9, None): + state = _run("山田太郎", policy=_JA, + segmenter=_fake((2,), confidence=conf)) + assert _texts(state) == ["山田", "太郎"] + assert state.ambiguities == () + + +def test_out_of_bounds_split_is_a_decline() -> None: + # Segmentation cannot check the upper bound (it never sees the + # text), so the stage does -- a violating answer is a buggy + # segmenter, and a buggy segmenter must not crash a parse + assert _texts(_run("山田", policy=_JA, segmenter=_fake((5,)))) == \ + ["山田"] + # 2 on a two-character token is the boundary itself: the check is + # >=, not >, since a cut at len(text) would leave an empty piece + assert _texts(_run("山田", policy=_JA, segmenter=_fake((2,)))) == \ + ["山田"] + + +def test_family_comma_gates_the_segmenter_too() -> None: + # the comma doctrine is about the input's structure, not the + # split's source: the structural opt-out runs before any consult + out = _run("山田太郎, 花子", policy=_JA, segmenter=_fake((2,))) + assert out.structure is Structure.FAMILY_COMMA + assert _texts(out) == ["山田太郎", "花子"] + assert out.ambiguities == () + + +def test_empty_vocabulary_still_consults_the_segmenter() -> None: + # "consult on vocabulary decline" has no unstated exception for a + # lexicon that declines EVERYTHING: a surname-less vocabulary is + # the JA pack's own shape (the segmenter is its whole mechanism), + # so an emptiness bail here would make it silently inert + state = _run("山田太郎", policy=_JA, lexicon=Lexicon.empty(), + segmenter=_fake((2,))) + assert _texts(state) == ["山田", "太郎"] + + +def test_a_wrong_answer_type_raises_a_curated_error() -> None: + # a duck-typed lookalike would otherwise wander into the split + # path and surface as a ValueError naming Token -- the reader's + # bug is in their segmenter, and the message must say so + class NotASegmentation: + splits = (2,) + confidence = None + + def seg(text: str) -> Segmentation | None: + return NotASegmentation() # type: ignore[return-value] + + with pytest.raises(TypeError, + match="segmenter must return Segmentation or None, " + "got NotASegmentation"): + _run("山田太郎", policy=_JA, segmenter=seg) + + diff --git a/tests/v2/pipeline/test_state.py b/tests/v2/pipeline/test_state.py index f927ec1..fe5c167 100644 --- a/tests/v2/pipeline/test_state.py +++ b/tests/v2/pipeline/test_state.py @@ -52,12 +52,15 @@ def test_stage_field_ownership() -> None: # a character offset that tokenize resolves to a token index "tokenize": {"tokens", "comma_offsets", "ambiguities"}, "segment": {"segments", "structure", "ambiguities"}, - # script_segment splits one unspaced CJK token in two, so it + # script_segment splits one unspaced CJK token into n+1 pieces + # (n = 1 from the vocabulary, any n from a segmenter), so it # rewrites tokens and shifts every later index the earlier # stages recorded -- the segment runs included. It is # deliberately absent from token_ownership below, whose # token-count assert is the one contract this stage is exempt - # from. structure it only READS (the FAMILY_COMMA opt-out). + # from. structure and segmenter it only READS (the + # FAMILY_COMMA opt-out, and the hook it consults on a + # vocabulary decline). "script_segment": {"tokens", "segments", "ambiguities"}, # classify also emits SUFFIX_OR_NICKNAME: the delimiter escape # that decides it lives in extract_delimited, which has no token diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index cca4176..85b6653 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -629,8 +629,6 @@ def test_parser_picklability_is_conditional_on_the_segmenter() -> None: pickle.dumps(unpicklable) # only pickling fails -@pytest.mark.xfail(strict=True, - reason="the stage consults the segmenter in the next commit") def test_segmenter_exceptions_propagate() -> None: # the ONE exception to parse-totality (locales spec section 4, # declared 2026-07-11): a user-supplied callable's own error is a @@ -641,4 +639,27 @@ def boom(text: str) -> Segmentation | None: p = parser_for(locales.get("zh"), base=Parser(segmenter=boom)) with pytest.raises(RuntimeError, match="segmenter bug"): p.parse("阿明") # zh pack active, 阿 unmatched by vocabulary -> - # the segmenter will be consulted once Task 4 lands + # the stage consults the segmenter + + +def test_a_segmenter_split_reaches_the_fields() -> None: + # end to end: the sub-slices the stage makes are ordinary tokens + # from there on, so the pack's family-first order reads them like + # any vocabulary split, and a low-confidence answer surfaces + def two(text: str) -> Segmentation | None: + return Segmentation((1,), confidence=0.5) + + p = parser_for(locales.get("zh"), base=Parser(segmenter=two)) + n = p.parse("阿明") + assert (n.family, n.given) == ("阿", "明") + assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] + + # two cuts, three pieces -- and the token indices the comma + # structure recorded still name the right words afterwards + def three(text: str) -> Segmentation | None: + return Segmentation((1, 2)) + + q = parser_for(locales.get("zh"), base=Parser(segmenter=three)) + n3 = q.parse("Dr 阿明日, Jr.") + assert (n3.title, n3.family, n3.given, n3.middle, n3.suffix) == ( + "Dr", "阿", "明", "日", "Jr.") diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 3940f83..b1c028b 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -8,7 +8,9 @@ from nameparser._locale import Locale from nameparser._parser import Parser from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch -from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token +from nameparser._types import ( + Ambiguity, AmbiguityKind, ParsedName, Role, Segmentation, Span, Token, +) def _decline(text: str, blob: str = "") -> None: @@ -117,6 +119,13 @@ def test_parser_repr_names_the_segmenter_without_its_contents() -> None: assert "segmenter=partial" in partial assert "xxx" not in partial and len(partial) < len(plain) + 20 + class Div: # a callable INSTANCE: no __qualname__, so the type + def __call__(self, text: str) -> Segmentation | None: + return None + + instance = repr(Parser(segmenter=Div())) + assert "segmenter=Div" in instance and "0x" not in instance + def test_policy_patch_repr_survives_unvalidated_name_order() -> None: # PolicyPatch defers name_order validation to apply time, so a diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 6904db4..12a17c7 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -1,5 +1,5 @@ import dataclasses -from collections.abc import Iterable +from collections.abc import Iterable, Iterator import pytest @@ -99,6 +99,23 @@ def test_segmentation_rejects_mappings_and_non_iterables() -> None: Segmentation(5) # type: ignore[arg-type] +def test_segmentation_lets_a_generator_raise_its_own_error() -> None: + # the non-iterable guard probes iter(), which cannot run a + # generator's body; wrapping the tuple() call instead would catch a + # TypeError raised INSIDE the user's generator and relabel it as + # "splits must be an iterable", pointing the reader at the wrong bug + def boom() -> Iterator[int]: + yield 1 + raise TypeError("boom") + + with pytest.raises(TypeError) as caught: + Segmentation(boom()) # type: ignore[arg-type] + # the message is the generator's own, whole -- a `match=` would + # pass on the relabeled text too, since the generator's repr names + # the function + assert str(caught.value) == "boom" + + def test_segmentation_is_frozen_and_hashable() -> None: s = Segmentation((2,)) assert isinstance(hash(s), int) From 00befa41f03b05ecc3e6a919ec2b0caf14befd49 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 04:08:47 -0700 Subject: [PATCH 06/14] Add the JA locale pack and the namedivider segmenter factory (#272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit locales.JA activates segmentation for the Japanese repertoire and locales.ja_segmenter() adapts namedivider-python (the new nameparser[ja] extra) to the Segmenter protocol; parser_for gains a segmenter= keyword so the two combine without a base Parser. Two findings from measuring namedivider 0.4.1 for real: * Its scores are bimodal, not clustered near 1 as the amendment assumed -- 1.0 for a rule-based division, a length-driven softmax in 0.23-0.68 for a statistical one, with the errors straddling the correct median. No floor inside that band separates error from success, so _SEGMENTER_CONFIDENCE_FLOOR stays 0.9, now sitting in the measured empty gap between the two modes rather than on a guess. * A segmenter must only be shown a whole name. Handed one token of an already-divided "山田 太郎" it answers anyway (scoring the two-character 山田 1.0 by rule), so the stage now consults it only when the gated token is the name part's only script-written one. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 29 ++ docs/locales.rst | 16 +- nameparser/_parser.py | 23 +- nameparser/_pipeline/_script_segment.py | 49 +- nameparser/locales/__init__.py | 21 + nameparser/locales/ja.py | 176 +++++++ pyproject.toml | 10 + tests/v2/test_layering.py | 17 +- tests/v2/test_locales.py | 207 +++++++- tests/v2/test_parser.py | 37 ++ uv.lock | 623 +++++++++++++++++++++++- 11 files changed, 1173 insertions(+), 35 deletions(-) create mode 100644 nameparser/locales/ja.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aacd72a..1ca6e0c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -61,3 +61,32 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml fail_ci_if_error: false + + # The optional nameparser[ja] segmenter (#272). Its own job, not a + # matrix entry: the matrix above must keep proving that the package + # installs and passes with NO third-party dependency at all, which is + # the promise the extra is carved out of. Everything the extra adds + # is under tests/v2/ (the ja pack's integration tests, which skip + # without it). + ja-extra: + runs-on: ubuntu-latest + # 3.14, not the matrix's newest: namedivider-python pulls lightgbm, + # which pulls scipy/numpy, and those have no 3.15 wheels yet (the + # sdist build fails). 3.15 is pre-release and this matrix is ahead + # of the ecosystem; revisit when it goes final. + env: + UV_PYTHON: "3.14" + steps: + - uses: actions/checkout@v7 + - name: Set up Python 3.14 + uses: actions/setup-python@v6 + with: + python-version: "3.14" + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + - name: Install dependencies + run: uv sync --group dev --extra ja + - name: Run Tests + run: uv run pytest tests/v2/ -q diff --git a/docs/locales.rst b/docs/locales.rst index 8dba47b..0bd9f87 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -109,7 +109,7 @@ takes: .. doctest:: >>> locales.available() - ('ru', 'tr_az', 'zh') + ('ja', 'ru', 'tr_az', 'zh') >>> locales.get("ru") is locales.RU True @@ -123,6 +123,14 @@ parsing, equivalent to ``parser_for(locales.get("ru"))``. * - Code - Turns on + * - ``ja`` + - Japanese segmentation — activates division for unspaced + Japanese names, which needs a segmenter to act: install + ``nameparser[ja]`` and pass + ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` + (``山田太郎`` → family ``山田``, given ``太郎``). The pack ships + no surname list, because no list divides a kanji name, and no + order: a Japanese name already reads family-first by default. * - ``ru`` - East Slavic patronymic order — detects a formal given/patronymic/family shape (Cyrillic and transliterated @@ -139,9 +147,9 @@ parsing, equivalent to ``parser_for(locales.get("ru"))``. name order: native-script Han already reads family-first without a pack. -``ru`` and ``tr_az`` are policy-only — they carry no vocabulary of -their own. ``zh`` is both halves at once: a surname list, plus the one -policy field that turns segmentation on for the script it covers. See +``ja``, ``ru`` and ``tr_az`` are policy-only — they carry no vocabulary +of their own. ``zh`` is both halves at once: a surname list, plus the +one policy field that turns segmentation on for the script it covers. See :doc:`concepts` for how that split (language vocabulary vs. behavior) is drawn, and `Contributing a pack to nameparser`_ for which half a new naming rule belongs in. diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 6aa615f..7625855 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -186,17 +186,22 @@ def parse(text: str) -> ParsedName: return _default_parser().parse(text) -def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: +def parser_for(*locales: Locale, base: Parser | None = None, + segmenter: Segmenter | None = None) -> Parser: """Lexicon fragments unioned left-to-right onto base's; policy patches applied left-to-right (later wins; set-valued fields union per the patch metadata). Validation errors raised while applying a pack are wrapped with that pack's identity (spec §4 amendment) -- PolicyPatch validates lazily, so with stacked packs the raw error would otherwise point at nothing. Two packs setting the same SCALAR - field is a declared conflict: UserWarning, later wins. base's - ``segmenter`` carries through unchanged -- packs are pure data and - cannot supply one, so ``base=Parser(segmenter=...)`` is how a pack - and a segmenter combine.""" + field is a declared conflict: UserWarning, later wins. + + A ``segmenter`` is passed straight through to the built Parser -- + ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` is how + a pack and a segmenter combine, since packs are pure data and + cannot supply one. Given explicitly it OVERRIDES base's (later + wins, the rule scalar policy fields follow); omitted, base's + carries through unchanged.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") for loc in locales: @@ -206,9 +211,11 @@ def parser_for(*locales: Locale, base: Parser | None = None) -> Parser: policy = base.policy if base is not None else Policy() # Unpacked here with its siblings because the return builds a FRESH # Parser: any field not listed there silently takes its default, and - # a dropped segmenter would be invisible (packs are pure data and - # cannot supply one, so base is its only source). - segmenter = base.segmenter if base is not None else None + # a dropped segmenter would be invisible. None reads as "not given" + # rather than "clear it" (the escape hatch is a plain Parser: + # Parser(lexicon=p.lexicon, policy=p.policy)). + if segmenter is None and base is not None: + segmenter = base.segmenter scalar_setters: dict[str, str] = {} for loc in locales: for f in dataclasses.fields(PolicyPatch): diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index e9cbe09..6288b0b 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -24,7 +24,12 @@ and wins, and the segmenter takes what is left. Its Segmentation may cut anywhere and any number of times, which is why the split path below is written for n cuts and the vocabulary hit is simply its -one-cut case. A segmenter's own exceptions PROPAGATE -- the single +one-cut case. One precondition guards it that the vocabulary has no +twin of: a segmenter answers where an UNDIVIDED name divides, so it is +consulted only when the gated token is the name part's ONLY +script-written one -- "山田 太郎" was divided by its writer and must +not have its family divided again (a Latin title or suffix draws no +such boundary). A segmenter's own exceptions PROPAGATE -- the single declared exception to parse totality (locales spec section 4): a user-supplied callable's error is a user-code error, not a content error. @@ -76,9 +81,25 @@ from nameparser._types import AmbiguityKind, Segmentation, Span #: Segmenter answers scoring below this attach a SEGMENTATION report -#: (amendment 2026-07-29 section 3). Placeholder pending the Task-5 -#: measurement against namedivider's real score distribution -- revisit -#: there, not here. Not configurable in 2.x (YAGNI). +#: (amendment 2026-07-29 section 3). Kept at the drafted 0.9 after +#: measuring namedivider 0.4.1 over 112 names (#272 Task 5), which +#: found a distribution the amendment did not anticipate: the scores +#: are BIMODAL, not clustered near 1. A rule-based division (the kana +#: boundary in 高橋みなみ, a two-character name) scores exactly 1.0; +#: a kanji-statistics division scores a softmax over the candidate cut +#: positions, observed in 0.23-0.68 and driven more by name LENGTH +#: (median 0.61 at three characters, 0.37 at five -- the softmax is +#: over len-1 candidates) than by correctness: the wrong answers in +#: the sample scored 0.32/0.60/0.60/0.61, straddling the correct +#: median of 0.52. So no floor INSIDE that band separates error from +#: success, and the only cut the data supports is between a stated +#: certainty and a statistical guess -- which is what section 3's +#: epistemic argument asks for anyway. 0.9 sits in the empty gap +#: (0.68 to 1.0), far from both modes, and reads as "confident" for a +#: third-party segmenter with a calibrated score too. Consequence, +#: pinned by tests: a kanji-only Japanese name always carries a +#: SEGMENTATION report, a kana-divided one never does. +#: Not configurable in 2.x (YAGNI). _SEGMENTER_CONFIDENCE_FLOOR = 0.9 @@ -227,6 +248,26 @@ def script_segment(state: ParseState) -> ParseState: return _split(state, i, (take,), detail) if state.segmenter is None: return state # the first activated-script token decides + # The segmenter's precondition, which the vocabulary has no twin of: + # it is asked where an UNDIVIDED name divides, so it may only be + # shown a token that is the whole name. Where the name part carries + # a second script-written token the writer already drew this + # stage's missing boundary -- "山田 太郎" is divided, and dividing + # its family again yields 山 + 田 + 太郎 (namedivider answers for + # any string, and scores a two-character one 1.0 by rule, so no + # confidence check would catch it). The neighbour counts whatever + # script it is written in, ACTIVATED or not -- effective_script is + # merely non-None -- because a katakana or hangul neighbour is a + # boundary its writer drew just as deliberately as a Han one: under + # the JA pack "山田太郎 マイケル" declines, though katakana is in no + # activation set. A Latin title or suffix is NOT such a boundary: + # it says nothing about where the CJK name splits, so "Dr 阿明日, + # Jr." still reaches the segmenter. Vocabulary keeps its own rule + # -- a listed surname is a certainty about that exact string, + # whoever else stands beside it. + if any(j != i and effective_script(state.tokens[j].text) is not None + for j in state.segments[0]): + return state # No try/except around the call: the module docstring's totality # exception. The check below is that same doctrine, curated -- a # duck-typed answer carrying a .splits of its own would otherwise diff --git a/nameparser/locales/__init__.py b/nameparser/locales/__init__.py index 40982b1..1f45ec9 100644 --- a/nameparser/locales/__init__.py +++ b/nameparser/locales/__init__.py @@ -12,12 +12,17 @@ from __future__ import annotations import importlib +from typing import TYPE_CHECKING from nameparser._locale import Locale +if TYPE_CHECKING: + from nameparser._types import Segmenter + #: attribute name -> (module name, module attribute). Codes are the #: lowercase module names; attribute constants are uppercase (spec §2). _REGISTRY = { + "JA": ("nameparser.locales.ja", "JA"), "RU": ("nameparser.locales.ru", "RU"), "TR_AZ": ("nameparser.locales.tr_az", "TR_AZ"), "ZH": ("nameparser.locales.zh", "ZH"), @@ -51,6 +56,22 @@ def available() -> tuple[str, ...]: return tuple(sorted(attr.lower() for attr in _REGISTRY)) +def ja_segmenter(*, gbdt: bool = False) -> Segmenter: + """The Japanese segmenter factory (#272), needing the optional + ``pip install nameparser[ja]``; see :mod:`nameparser.locales.ja` for + what it wraps, what it declines, and its licensing. + + A real function rather than a _REGISTRY entry, because the registry + resolves LOCALES: __getattr__ is annotated to return one, so routing + a callable through it would type every pack access as "Locale or + Segmenter" for no gain. Laziness is preserved here instead -- the + pack module (and with it the optional dependency) loads on the + call, not on importing this package.""" + from nameparser.locales.ja import ja_segmenter as factory + + return factory(gbdt=gbdt) + + def get(code: str) -> Locale: """Dynamic lookup by code ('ru'); raises KeyError listing the available codes (spec §2).""" diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py new file mode 100644 index 0000000..e0e215f --- /dev/null +++ b/nameparser/locales/ja.py @@ -0,0 +1,176 @@ +"""The Japanese locale pack (#272): activates segmentation for the +Japanese repertoire -- Han and the kana-licensed composites that +resolve to HIRAGANA -- so that an unspaced 山田太郎 can be divided at +all. Applying this pack IS the "my data is Japanese" declaration, the +same role locales.ZH plays for Chinese: a pure-Han string cannot say +which language it is written in (林 is Lin or Hayashi), so neither +activation may be a default. + +Unlike zh, this pack ships NO vocabulary and NO order: + +* No vocabulary, because no surname list divides Japanese names. Family + and given names draw on the same kanji, both sides run one to four + characters, and the reading -- not the spelling -- is what settles + most splits. That is what the pluggable segmenter is for, and it is + why this pack alone is inert: it activates a stage that then has + nothing to segment with. +* No order, because a kana-bearing Japanese name already reads + family-first by default (amendment 2026-07-29 §1: hiragana identifies + native Japanese as certainly as hangul identifies Korean), and wholly + Han names have read family-first since the 2026-07-27 amendment. + Pure KATAKANA is deliberately outside both the order rule and the + activation set above -- katakana is how Japanese writes FOREIGN names + (マイケル・ジャクソン), in their original given-first order. + +Data sources: none. The pack carries no vocabulary of its own, so +there is no list here to cite, extend or keep current -- the data that +divides a Japanese name belongs to the segmenter, and namedivider's +sources and their terms are set out under Licensing below. + +The segmenter is a FACTORY, ``ja_segmenter()``, not pack data: a Locale +is pure data that dissolves into a Parser at construction, and a +third-party callable is neither pure nor data. Combining the two is +explicit -- ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` +-- which also keeps the optional dependency out of every import path +that merely touches this module. + +Licensing (the optional dependency is never bundled; nameparser's own +LGPL is untouched): namedivider-python's source and its GBDT model are +MIT. Its surname data -- ``family_name_repository.pickle``, sourced +from Myoji-Yurai.net -- carries custom terms permitting commercial and +non-commercial use *for dividing names*, which is the only use any +caller of this module can put it to. Those terms bind only the opt-in +``ja_segmenter(gbdt=True)`` path, which is what loads that file; the +default BasicNameDivider reads namedivider's own bundled kanji.csv and +never touches it. Its BERT model for katakana division is CC-BY-SA and +is NOT used (katakana division is out of scope, amendment §6). + +Declared deviations (spec §2 authoring requirement 3): the pack sets +one union policy field, self-selecting by script, so it can only change +names containing characters of the Japanese repertoire -- DEVIATES +below declares exactly that, over-declaring within the repertoire (a +name also needs an unspaced token and a segmenter to actually change, +but per-character scanning is the safe direction, zh's precedent). +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from nameparser._lexicon import Lexicon +from nameparser._locale import Locale +from nameparser._policy import PolicyPatch, Script +from nameparser._types import Segmentation + +if TYPE_CHECKING: + from nameparser._types import Segmenter + +JA = Locale( + code="ja", + lexicon=Lexicon.empty(), + policy=PolicyPatch( + segment_scripts=frozenset({Script.HAN, Script.HIRAGANA})), +) + +# The Japanese repertoire's codepoint spans -- hiragana, katakana and +# Han -- kept in sync BY HAND with _pipeline/_vocab.py's _SCRIPT_RANGES +# entries for those three scripts (layering forbids a pack importing +# the pipeline; the sync test in tests/v2/test_locales.py pins the +# equality). Katakana is in the table even though the pack never +# ACTIVATES it: both consumers below quantify over the whole +# repertoire, because a katakana-bearing token can still be a +# kana-licensed composite the pack acts on (山田エミ) -- DEVIATES must +# declare it, and the adapter must not decline it. A pure-katakana +# token never reaches either: the stage's own gate stops it. +_JA_RANGES = ((0x3040, 0x309F), (0x30A0, 0x30FF), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), + (0x20000, 0x323AF)) + + +def _in_repertoire(char: str) -> bool: + return any(lo <= ord(char) <= hi for lo, hi in _JA_RANGES) + + +def DEVIATES(name: str) -> bool: + """True when this pack may parse `name` differently from the + default parser (the declared-deviation predicate the + non-interference gate consumes).""" + return any(_in_repertoire(c) for c in name) + + +def ja_segmenter(*, gbdt: bool = False) -> Segmenter: + """A :data:`~nameparser.Segmenter` wrapping namedivider-python, + installed with ``pip install nameparser[ja]``: + + parser = parser_for(locales.JA, segmenter=locales.ja_segmenter()) + + Wraps ``BasicNameDivider`` (99.3% accurate, ~4k names/sec, pure + Python) by default, which needs only data bundled in the installed + package; ``gbdt=True`` selects ``GBDTNameDivider`` (99.9%), which + asks more of the environment on two counts. It runs namedivider's + gradient-boosted model through lightgbm -- a hard dependency of + namedivider-python, so the extra installs it, but its wheel does + not bundle the native OpenMP runtime it loads at import (a missing + ``libomp`` surfaces as lightgbm's own OSError naming the library it + could not find). And the first such call DOWNLOADS its model and + surname files from the network into ``~/.cache/namedivider-python`` + (namedivider prints its own progress to stdout doing so), which + matters for offline, sandboxed or air-gapped deployments. + + The adapter declines -- returns None, leaving the token whole -- + for text outside the Japanese repertoire, for text too short to + divide, and for any answer that fails to reconstruct its input. An + answer that puts the whole token on one side comes back as + "confidently one token" rather than a decline: the divider read it + and found nothing to cut, which is not the same as having no + opinion. + Answers arrive with namedivider's own confidence, which the parsing + stage compares against its floor to decide whether to report a + SEGMENTATION ambiguity: namedivider scores a rule-based division + (the kana boundary in 高橋みなみ, a two-character name) 1.0 and a + kanji-statistics division well below that, so in practice every + statistically-divided name carries the report and every + rule-divided one does not. + """ + try: + import namedivider + except ImportError as exc: + raise ImportError( + "the Japanese segmenter needs namedivider-python -- " + "install it with: pip install nameparser[ja]" + ) from exc + divider = (namedivider.GBDTNameDivider() if gbdt + else namedivider.BasicNameDivider()) + + def segment(text: str) -> Segmentation | None: + # segment_scripts UNIONS under pack composition, so this can + # receive tokens of any other activated script (hangul, under + # the default policy). Decline anything outside the Japanese + # repertoire rather than hand it to namedivider, which would + # answer for it regardless -- the same table DEVIATES scans. + if not all(map(_in_repertoire, text)): + return None + # namedivider RAISES below two characters ("Name length needs + # at least 2 chars"), and a segmenter's exceptions propagate by + # contract, so a one-character token -- routine in spaced input + # like "林 太郎" -- must be declined here, not passed on. + if len(text) < 2: + return None + result = divider.divide_name(text) + if result.family + result.given != text: + return None # defensive: never trust reconstruction + # float(): the score is a numpy scalar on the statistical path, + # and clamping guards the [0, 1] range Segmentation enforces + # against float error at the edge of namedivider's softmax (a + # violation would raise mid-parse). + confidence = min(1.0, max(0.0, float(result.score))) + # Defensive, the reconstruction check's twin: no namedivider + # 0.4.x path can return an empty side (every answer, rule or + # statistical, is built by slicing the input at an interior + # offset). If one ever did, it would mean "I read this as + # undivided" -- an empty Segmentation, which is a stated + # opinion, and not the None that declines to have one. + if not result.family or not result.given: + return Segmentation((), confidence=confidence) + return Segmentation((len(result.family),), confidence=confidence) + + return segment diff --git a/pyproject.toml b/pyproject.toml index 66a5159..2a95ea8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,13 @@ classifiers = [ ] dependencies = [] +[project.optional-dependencies] +# Japanese kanji-name segmentation (#272), wrapped by +# locales.ja_segmenter(). Optional and never bundled: the core stays +# zero-dependency. 0.4 is the floor its BasicNameDivider/DividedName +# API dates from. +ja = ["namedivider-python>=0.4"] + [build-system] requires = ["setuptools (>=82.0.1)"] build-backend = "setuptools.build_meta" @@ -53,6 +60,9 @@ packages = ["nameparser", "tests"] [[tool.mypy.overrides]] module = [ "dill.*", + # optional extras, unstubbed and absent from the dev environment: + # the ja pack imports namedivider lazily, inside the factory + "namedivider.*", ] ignore_missing_imports = true diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index 081b7e7..f4495e7 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -19,8 +19,8 @@ "_pipeline/_classify.py", "_pipeline/_group.py", "_pipeline/_assign.py", "_pipeline/_post_rules.py", "_pipeline/_assemble.py", "_parser.py", "_facade.py", - "_config_shim.py", "locales/__init__.py", "locales/ru.py", - "locales/tr_az.py"} + "_config_shim.py", "locales/__init__.py", "locales/ja.py", + "locales/ru.py", "locales/tr_az.py", "locales/zh.py"} _PIPELINE_STAGE_ALLOWED = ( "nameparser._types", "nameparser._lexicon", "nameparser._policy", @@ -29,11 +29,16 @@ "nameparser._pipeline.", ) -# a locale pack: pure data over the three base types, no pipeline or -# config access (locales spec §2) -- one contract shared by every pack, -# like _PIPELINE_STAGE_ALLOWED above, so tightening it is a one-line edit +# a locale pack: pure data over the base types, no pipeline or config +# access (locales spec §2) -- one contract shared by every pack, like +# _PIPELINE_STAGE_ALLOWED above, so tightening it is a one-line edit. +# _types joined the three in #272: the ja pack's segmenter factory +# constructs a Segmentation, and _types is the bottom of the graph +# (Locale itself imports it), so this widens the base types, not the +# reach. _LOCALE_PACK_ALLOWED = ( "nameparser._lexicon", "nameparser._locale", "nameparser._policy", + "nameparser._types", ) # module -> prefixes it may import from within nameparser @@ -91,8 +96,10 @@ # "nameparser.locales." prefix documents that reach for humans even # though the walker never exercises it. "locales/__init__.py": ("nameparser._locale", "nameparser.locales."), + "locales/ja.py": _LOCALE_PACK_ALLOWED, "locales/ru.py": _LOCALE_PACK_ALLOWED, "locales/tr_az.py": _LOCALE_PACK_ALLOWED, + "locales/zh.py": _LOCALE_PACK_ALLOWED, # v1 import-path preservation: thin re-exports of the facade/shim "parser.py": ("nameparser._facade",), "config/__init__.py": ("nameparser._config_shim",), diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index b15385d..0ac5213 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -1,7 +1,7 @@ """The locale pack layer (locales spec §2-3): lazy access, the shipped packs, composition, and the non-interference gate.""" import functools -import importlib +import importlib.util import re from collections.abc import Callable, Iterable from types import ModuleType @@ -13,6 +13,7 @@ from nameparser._pipeline._vocab import _SCRIPT_RANGES from nameparser._policy import UNSET, PatronymicRule, Policy, Script from nameparser._types import AmbiguityKind +from nameparser.locales import ja as _ja from nameparser.locales import ru as _ru from nameparser.locales import tr_az as _tr_az from nameparser.locales import zh as _zh @@ -36,11 +37,46 @@ def _pack_modules() -> dict[str, ModuleType]: _PACKS = _pack_modules() +# ja is the one pack whose behavior is not pack data: it activates +# segmentation and supplies nothing to segment WITH, so every parse +# through it needs the optional nameparser[ja] segmenter. Probed once +# here; without the extra the ja code has no _PACKED entry, so every +# test that PARSES through the pack skips -- its rotator rows and its +# non-interference run included. What still runs either way is what +# needs no parse: the registry contract (DEVIATES and a rotator list +# exist), the range pin, the pack's contents, and the inertness test +# below, which is the unsegmented pack's whole observable behavior. +_JA_AVAILABLE = importlib.util.find_spec("namedivider") is not None +_needs_ja = pytest.mark.skipif(not _JA_AVAILABLE, reason="needs nameparser[ja]") + # shared across the gate and rotator tests: the default-parser baseline # is identical everywhere (only the packed side varies per test), so # parse each name once for the whole module instead of once per test _DEFAULT_PARSER = Parser() -_PACKED = {code: parser_for(locales.get(code)) for code in _PACKS} + + +def _packed_parsers() -> dict[str, Parser]: + out: dict[str, Parser] = {} + for code in _PACKS: + if code != "ja": + out[code] = parser_for(locales.get(code)) + elif _JA_AVAILABLE: + out[code] = parser_for( + locales.JA, segmenter=locales.ja_segmenter()) + return out + + +_PACKED = _packed_parsers() + + +def _require_packed(code: str) -> Parser: + """The pack's parser, skipping the test when it needs an optional + extra that is not installed (ja only, today).""" + parser = _PACKED.get(code) + if parser is None: + pytest.skip(f"the {code!r} pack needs nameparser[{code}], " + f"which is not installed") + return parser @functools.cache @@ -58,9 +94,13 @@ def test_locales_module_attribute_access() -> None: def test_locales_get_and_available() -> None: assert locales.get("ru") is locales.RU - assert set(locales.available()) == {"ru", "tr_az", "zh"} - with pytest.raises(KeyError, match="ru, tr_az, zh"): + assert set(locales.available()) == {"ja", "ru", "tr_az", "zh"} + with pytest.raises(KeyError, match="ja, ru, tr_az, zh"): locales.get("xx") + # the segmenter FACTORY is not a locale: it is an ordinary function + # on the package, so it must not appear among the codes + assert "ja_segmenter" not in locales.available() + assert callable(locales.ja_segmenter) def test_tr_az_pack_contents() -> None: @@ -156,6 +196,135 @@ def test_zh_han_ranges_stay_in_sync_with_vocab() -> None: assert _zh._HAN_RANGES == _SCRIPT_RANGES[Script.HAN] +def test_ja_pack_contents() -> None: + assert locales.JA.code == "ja" + # segmentation activation ONLY: no vocabulary (no list settles a + # kanji name) and no order (the kana license already reads + # Japanese family-first by default, amendment 2026-07-29 §1) + assert locales.JA.policy.segment_scripts == frozenset( + {Script.HAN, Script.HIRAGANA}) + assert locales.JA.policy.name_order is UNSET + assert locales.JA.policy.script_orders is UNSET + assert locales.JA.lexicon == Lexicon.empty() + + +def test_ja_segmenter_without_extra_raises_helpfully() -> None: + if _JA_AVAILABLE: + pytest.skip("namedivider installed; the error path is untestable") + with pytest.raises(ImportError, match=r"nameparser\[ja\]"): + locales.ja_segmenter() + + +def test_ja_ranges_stay_in_sync_with_vocab() -> None: + # the twin of the zh pin above: ja.py hand-copies the three + # Japanese-repertoire scripts' spans for DEVIATES and for the + # adapter's own repertoire check + assert set(_ja._JA_RANGES) == { + span for script in (Script.HAN, Script.HIRAGANA, Script.KATAKANA) + for span in _SCRIPT_RANGES[script]} + + +def test_ja_pack_alone_is_inert() -> None: + # The pack's own claim, and the reason it is safe to register + # without the extra: activating segmentation while shipping nothing + # to segment WITH changes no parse at all. Pure pack data, so this + # runs whether or not nameparser[ja] is installed -- and it is the + # unsegmented pack's entire observable behavior. + p = parser_for(locales.JA) + for name in _ROTATORS["ja"]: + assert p.parse(name).as_dict() == _default_parse(name), name + + +@_needs_ja +def test_ja_end_to_end() -> None: + p = _PACKED["ja"] + n = p.parse("山田太郎") + assert (n.family, n.given) == ("山田", "太郎") + # the case a zh surname list corrupts (高 + 橋一郎), done right + n = p.parse("高橋一郎") + assert (n.family, n.given) == ("高橋", "一郎") + # kana-licensed composite: gated in through the HIRAGANA entry + n = p.parse("高橋みなみ") + assert (n.family, n.given) == ("高橋", "みなみ") + + +@_needs_ja +def test_ja_keeps_a_transcribed_name_in_its_source_order() -> None: + # pure katakana is excluded from the pack's activation by design + # (amendment §1: katakana is how Japanese writes FOREIGN names), so + # the nakaguro form divides on the dot and keeps its given-first + # source order instead of being read family-first + n = _PACKED["ja"].parse("マイケル・ジャクソン") + assert (n.given, n.family) == ("マイケル", "ジャクソン") + + +@_needs_ja +def test_ja_composes_with_zh() -> None: + # vocabulary first, segmenter on decline (amendment §2) + p = parser_for(locales.ZH, locales.JA, segmenter=locales.ja_segmenter()) + assert p.parse("毛泽东").family == "毛" # zh surname wins + assert p.parse("山田太郎").family == "山田" # zh declines, segmenter + # ...and a Korean name still splits on the default vocabulary, + # never reaching the segmenter (which would decline it anyway) + assert p.parse("김민준").family == "김" + + +@_needs_ja +def test_ja_segmenter_declines_what_it_cannot_read() -> None: + # The adapter's repertoire guard, which only a token that actually + # REACHES the adapter can pin: a listed Korean surname is split by + # the vocabulary first (the second case below), so this hangul + # deliberately opens with a syllable no surname list carries. It + # gates in on script, declines vocabulary, and reaches the adapter, + # which must decline it rather than let namedivider divide Korean + # -- without the guard it comes back as family 쿄 carrying a bogus + # SEGMENTATION report. + p = _PACKED["ja"] + n = p.parse("쿄쿄쿄") + assert (n.family, n.given) == ("쿄쿄쿄", "") + assert n.ambiguities == () + # ...and the vocabulary-wins case, which never reaches the adapter + assert p.parse("김민준").family == "김" + + +@_needs_ja +def test_ja_segmenter_declines_a_lone_character() -> None: + # namedivider RAISES below two characters, and a segmenter's + # exceptions propagate by contract, so the adapter's length guard + # is what keeps this parse total. It takes a LONE token to reach: + # in "林 太郎" the stage's precondition stops the segmenter first. + p = _PACKED["ja"] + n = p.parse("林") + assert (n.family, n.given) == ("林", "") + assert n.ambiguities == () + + +@_needs_ja +def test_ja_leaves_spaced_names_to_the_default_order() -> None: + # the segmenter's precondition (see test_parser.py): a spaced name + # is already divided, and the kana/Han family-first defaults read + # it correctly with no segmenter involved. Without this the pack + # would wreck the commonest written form -- 山田 + 太郎 divided + # again into 山 + 田 + 太郎. + p = _PACKED["ja"] + for name in ("山田 太郎", "高橋 みなみ", "高橋 エミ", "山田 太郎 Jr."): + assert p.parse(name).as_dict() == _default_parse(name), name + assert p.parse("山田 太郎").family == "山田" + + +@_needs_ja +def test_ja_reports_statistical_divisions_and_not_rule_based_ones() -> None: + # the confidence floor's measured consequence (_script_segment.py's + # constant): namedivider scores a rule-based division 1.0 and a + # kanji-statistics division far below the floor, so the ambiguity + # report separates exactly those two epistemic states + p = _PACKED["ja"] + assert [a.kind for a in p.parse("山田太郎").ambiguities] == [ + AmbiguityKind.SEGMENTATION] + assert p.parse("高橋みなみ").ambiguities == () # kana boundary rule + assert p.parse("原恵").ambiguities == () # two-character rule + + def test_pack_vocabulary_entries_are_single_words() -> None: # The shipped-vocabulary guard (test_lexicon.py's # test_default_lexicon_builds_warning_free), extended to the OTHER @@ -394,6 +563,20 @@ def _default_corpus() -> list[str]: "諸葛亮", # traditional-script compound "萧红", # the 肖/萧 merger's second simplified spelling ] +# ja has no marker regexes and no vocabulary either: its rotators cover +# the shapes only the pack PLUS its segmenter changes. Every row is +# UNSPACED on purpose -- a spaced kana-licensed name (高橋 みなみ) now +# reads family-first by DEFAULT (amendment 2026-07-29 §1), so it would +# not deviate and would fail the must-deviate assertion below. +_ROTATORS["ja"] = [ + "山田太郎", # 2-kanji family + 2-kanji given, the common shape + "高橋一郎", # the name a zh surname list corrupts (高 + 橋一郎) + "田中花子", # ditto (田 is a listed zh surname) + "原辰徳", # 1-kanji family + 2-kanji given: the other fork + "高橋みなみ", # kanji + hiragana: the kana license's carrier case + "山田エミ", # kanji + katakana: licensed as a MIXED token, where + # pure katakana (マイケル) is deliberately not +] def _marker_regexes(module: ModuleType) -> list[re.Pattern]: @@ -478,25 +661,33 @@ def test_rotator_deviates_and_declares(code: str, name: str) -> None: # every branch of the pack's marker regexes both FIRES (packed parse # differs from default) and is DECLARED (DEVIATES says so) -- the # per-name proof behind the gate's declared-count floor below - assert _PACKED[code].parse(name).as_dict() != _default_parse(name) + packed = _require_packed(code) + assert packed.parse(name).as_dict() != _default_parse(name) assert _PACKS[code].DEVIATES(name) @pytest.mark.parametrize("code", sorted(_PACKS)) def test_non_interference_each_pack(code: str) -> None: + packed = _require_packed(code) corpus = _default_corpus() + _ROTATORS[code] declared = _assert_non_interference( - _PACKED[code], _PACKS[code].DEVIATES, corpus) + packed, _PACKS[code].DEVIATES, corpus) # the positive side: every synthetic rotator (plus the corpus's own # in-scope names) must actually flow through the declared branch assert declared >= len(_ROTATORS[code]) def test_non_interference_all_packs_combined() -> None: - all_rotators = [n for code in sorted(_ROTATORS) + # Every pack is applied whether or not its extra is installed (the + # ja pack's policy patch is pure data); only the ROTATORS of a pack + # that cannot act are dropped, since the count assertion below + # requires each listed name to actually deviate. + all_rotators = [n for code in sorted(_ROTATORS) if code in _PACKED for n in _ROTATORS[code]] corpus = _default_corpus() + all_rotators - packed = parser_for(*(locales.get(code) for code in sorted(_PACKS))) + packed = parser_for( + *(locales.get(code) for code in sorted(_PACKS)), + segmenter=locales.ja_segmenter() if _JA_AVAILABLE else None) declared = _assert_non_interference( packed, lambda n: any(m.DEVIATES(n) for m in _PACKS.values()), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 85b6653..edbcafc 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -615,10 +615,31 @@ def test_parser_segmenter_is_keyword_only_and_validated() -> None: def test_parser_for_carries_the_base_segmenter() -> None: def seg(text: str) -> Segmentation | None: return None + # not given: the base's carries through unchanged p = parser_for(locales.get("zh"), base=Parser(segmenter=seg)) assert p.segmenter is seg +def test_parser_for_takes_a_segmenter_keyword() -> None: + def seg(text: str) -> Segmentation | None: + return None + p = parser_for(locales.get("zh"), segmenter=seg) + assert p.segmenter is seg + assert parser_for(locales.get("zh")).segmenter is None + + +def test_parser_for_segmenter_keyword_overrides_the_base() -> None: + # later wins, the same rule scalar policy fields follow + def from_base(text: str) -> Segmentation | None: + return None + + def explicit(text: str) -> Segmentation | None: + return None + p = parser_for(locales.get("zh"), base=Parser(segmenter=from_base), + segmenter=explicit) + assert p.segmenter is explicit + + def test_parser_picklability_is_conditional_on_the_segmenter() -> None: # declared contract (locales spec section 4): Parser pickles iff # its segmenter does -- like any callable-holding object @@ -642,6 +663,22 @@ def boom(text: str) -> Segmentation | None: # the stage consults the segmenter +def test_the_segmenter_sees_only_an_undivided_name() -> None: + # its precondition (#272 Task 5): a segmenter answers where an + # UNDIVIDED name divides, so a name part carrying a second + # script-written token is already divided and the segmenter is not + # consulted -- otherwise "山田 太郎" would have its family divided + # again. A Latin title/suffix is not such a boundary. + def always(text: str) -> Segmentation | None: + return Segmentation((1,), confidence=1.0) + + p = parser_for(locales.get("zh"), base=Parser(segmenter=always)) + assert p.parse("阿明").family == "阿" # one token: consulted + n = p.parse("阿明 日月") # already divided + assert (n.family, n.given) == ("阿明", "日月") + assert p.parse("Dr 阿明").family == "阿" # a title is not a split + + def test_a_segmenter_split_reaches_the_fields() -> None: # end to end: the sub-slices the stage makes are ordinary tokens # from there on, so the pack's family-first order reads them like diff --git a/uv.lock b/uv.lock index 7ac65e1..ba1e21c 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,18 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version >= '3.12' and python_full_version < '3.15'", - "python_full_version < '3.12'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] [[package]] @@ -28,6 +37,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "ast-serialize" version = "0.5.0" @@ -494,6 +512,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/62/b40b382fa0c66fee1478073eb8db352a4a6beda4a1adccf1df911d8c289c/librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", size = 102572, upload-time = "2026-05-10T18:17:06.809Z" }, ] +[[package]] +name = "lightgbm" +version = "4.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/8e/4db5e29290d7e619c307fdb8dab0a0514090af2ce3ec483050e024ec6126/lightgbm-4.7.0.tar.gz", hash = "sha256:f8e20f682c9aabd000bcf4a7ed8aa6f473c1adfecccae34ec24e823d156f4af0", size = 1792896, upload-time = "2026-07-18T21:00:56.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/05/7213965863cba1ed0150ad045bceed6276a1afaaaedbaeff4699ec4f0ccb/lightgbm-4.7.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:dfc1cfe8e760387be1e7ba7a214688be21fdff96e4ed9749188f83e1877c2477", size = 1877851, upload-time = "2026-07-18T21:00:35.225Z" }, + { url = "https://files.pythonhosted.org/packages/b2/86/f4fe714f2e0bf3941705a20d7f6849dc476276d71236e82ea6b0d6539b86/lightgbm-4.7.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:129535462686f274df179133643118c5c5c5667167fe6c3a28d955f0b3c8e868", size = 1498914, upload-time = "2026-07-18T21:00:36.549Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/b29580948b92e8c2f84dea70118ac702ff067dc52ec4ffb5d73c953536a5/lightgbm-4.7.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4529acec5c6fefe4768302a529707d0ead90f6a6f42df694b856212e09695b8", size = 3349492, upload-time = "2026-07-18T21:00:37.943Z" }, + { url = "https://files.pythonhosted.org/packages/15/eb/837ea3b40cc36e22eeebb9785c01e42b2c255d033eea1d2d9ee8e2540e55/lightgbm-4.7.0-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d23e922acd891e77212e4d0fbcee9ba973c96dee479491341d05ba595357ebb7", size = 3476028, upload-time = "2026-07-18T21:00:39.331Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0b/c5c17d862b12ce292f24cd85d40f2f8f8981668fbdbd43fdc2625eccbc79/lightgbm-4.7.0-py3-none-win_amd64.whl", hash = "sha256:f42d1e5b32b6f170e606d7c689c6165671da98d7bf37f1addec2623efc8740c9", size = 1360833, upload-time = "2026-07-18T21:00:40.865Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -568,6 +618,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mypy" version = "2.1.0" @@ -628,10 +687,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "namedivider-python" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lightgbm" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pandas" }, + { name = "regex" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/e4/69d1a59c468d4ea1b80ce2f598de9994f5da7fcff35889b402fdf870e74f/namedivider_python-0.4.1.tar.gz", hash = "sha256:ef5d0aecc71fbf020da2d8b84793109d48aaf65da9d244f4746973c4821b08c0", size = 35271, upload-time = "2025-11-03T04:45:49.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/b7/8d7aaef9fb1480ab77030c0189929bd5489d848fb390ce945223def2063d/namedivider_python-0.4.1-py2.py3-none-any.whl", hash = "sha256:efedf5cfcddc6fcc46c8116cc883955888d9eed6d518169fe6af1e1683055b3f", size = 46427, upload-time = "2025-11-03T04:45:50.761Z" }, +] + [[package]] name = "nameparser" source = { editable = "." } +[package.optional-dependencies] +ja = [ + { name = "namedivider-python" }, +] + [package.dev-dependencies] dev = [ { name = "dill" }, @@ -647,6 +728,8 @@ dev = [ ] [package.metadata] +requires-dist = [{ name = "namedivider-python", marker = "extra == 'ja'", specifier = ">=0.4" }] +provides-extras = ["ja"] [package.metadata.requires-dev] dev = [ @@ -661,6 +744,161 @@ dev = [ { name = "sphinx", specifier = ">=8" }, ] +[[package]] +name = "narwhals" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/58946e5aab18393e793bd4add6985b95d0e01c3a2d832f38f54468b10dcd/narwhals-2.24.0.tar.gz", hash = "sha256:b5c0f684ccd9d7475b564111e319a4964abcf2baf79d3cf6b1003d06ac9b828d", size = 661143, upload-time = "2026-07-13T10:49:19.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/85/a5bfaebfd305ac18b57b0854d74e37e586809061a91fda62f0bd50c8518e/narwhals-2.24.0-py3-none-any.whl", hash = "sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489", size = 461030, upload-time = "2026-07-13T10:49:17.571Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -670,6 +908,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ef/f1fd7431d635bf20015489bf0bd69c17fff1018de773540f651455a3916b/pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282", size = 10397178, upload-time = "2026-07-22T22:17:48.274Z" }, + { url = "https://files.pythonhosted.org/packages/31/b4/0eafac990a431561187694126de01f9b12559549b4d86360c0c4bd870fde/pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298", size = 9990736, upload-time = "2026-07-22T22:17:52.388Z" }, + { url = "https://files.pythonhosted.org/packages/de/21/359880af3ea9b7cb23bea5b51e8e70ef3866c03be09da9a2787e18e330a8/pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899", size = 10814438, upload-time = "2026-07-22T22:17:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/d6cc4d7e508bbccf5d6027314a8312bc7ac73d0ec7f195f53838daafab40/pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a", size = 11323634, upload-time = "2026-07-22T22:17:56.858Z" }, + { url = "https://files.pythonhosted.org/packages/70/2b/d5f0a8c90dd0ae04e64ba53b871afb796ec026b615086d382ddc2ade729b/pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928", size = 11850860, upload-time = "2026-07-22T22:17:59.1Z" }, + { url = "https://files.pythonhosted.org/packages/5c/30/183aec2e19adf778a98d29b5729a0a68f4cc4ebf9b9c3b70d0297355bcb1/pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d", size = 12411100, upload-time = "2026-07-22T22:18:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/fa/9a/31f4983f191af51ab2a8f2d0c7b33dff3a84da26533f982fff02c2f9e28b/pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a", size = 9968804, upload-time = "2026-07-22T22:18:03.903Z" }, + { url = "https://files.pythonhosted.org/packages/49/97/7886c89a39045c69ad82cbceaf3343810480c8ef49a216319ce8183860a6/pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a", size = 9205447, upload-time = "2026-07-22T22:18:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "pathspec" version = "1.1.1" @@ -739,6 +1032,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e5/cef4de2bac939280b68d32adc659478845238a8274f2f79c465063f590ad/regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c", size = 494012, upload-time = "2026-07-19T00:16:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/ff/87/e86f51eb117457bb7803132ffe5cb6e2841e2b5bea4cc85d397f3c6e257d/regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae", size = 295281, upload-time = "2026-07-19T00:16:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/2360c41d8080a3d9ec7e5c90fad6eab3b50192869d10e9a5609e48c8177b/regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5", size = 290615, upload-time = "2026-07-19T00:16:43.058Z" }, + { url = "https://files.pythonhosted.org/packages/cf/69/b65ba4344efbc771b28fe5dde84cbbb6c8f9551165952fe78def5b9dde6a/regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20", size = 791804, upload-time = "2026-07-19T00:16:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/81/b6/a40dfa0dc6224b36f620c00296eacc830489cbf8c2837b6750dfe6170375/regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a", size = 861723, upload-time = "2026-07-19T00:16:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/02/735991dee71abd83196a7962f7ed8bf5aa05720ff06e2d3ff896a85e2bbb/regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7", size = 905932, upload-time = "2026-07-19T00:16:47.956Z" }, + { url = "https://files.pythonhosted.org/packages/45/6c/e7098d8b846ccdbf431d8c081b61e496526a27a28094ed09e0dce21b3f54/regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc", size = 801407, upload-time = "2026-07-19T00:16:49.43Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/34b69274e2649bcc7d9b089c2b2983fb2632d8ecf667e359593be9072e79/regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864", size = 774448, upload-time = "2026-07-19T00:16:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/0a72247d025585fd3800b98e040b84d562a88af6303347100484849f4f01/regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af", size = 783297, upload-time = "2026-07-19T00:16:53.071Z" }, + { url = "https://files.pythonhosted.org/packages/b1/aa/c4f65ae7dd02a36b323a70c4cff326e1f3442361aaebc9311100a130d54f/regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5", size = 854736, upload-time = "2026-07-19T00:16:54.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/668082bcc817b9e694189b84997aeba7385b7779faa6711788679c482e35/regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82", size = 763298, upload-time = "2026-07-19T00:16:56.289Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fb/2d07ad555e7af88aa5f867fdafa47a8d945ee237c20af3ebceb46a820835/regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78", size = 844430, upload-time = "2026-07-19T00:16:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/51/15/c82a471fe3dce56f03745635b43aa456c40dc0db089e07ef148b331507d1/regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327", size = 789683, upload-time = "2026-07-19T00:16:59.583Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/7532a2c59d56f5398902c20de60f0c9a5d1cd364e42a051b48e1b210be7b/regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d", size = 266778, upload-time = "2026-07-19T00:17:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/cf1bc631db154eb95520d9d5dbc2371ff77a0f014bbf7d748fed8496aa63/regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965", size = 277983, upload-time = "2026-07-19T00:17:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/8d/bd/56ceaf170e875d5a6761bf2bfd0d040f1cacc896850d5e40cb29b11bbd06/regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e", size = 276961, upload-time = "2026-07-19T00:17:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -754,6 +1163,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "roman-numerals" version = "4.1.0" @@ -788,6 +1210,162 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, ] +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/75/b4ce781849931fef6fd529afa6b63711d5a733065722d0c3e2724af9e40a/scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec", size = 31613675, upload-time = "2026-02-23T00:16:00.13Z" }, + { url = "https://files.pythonhosted.org/packages/f7/58/bccc2861b305abdd1b8663d6130c0b3d7cc22e8d86663edbc8401bfd40d4/scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696", size = 28162057, upload-time = "2026-02-23T00:16:09.456Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ee/18146b7757ed4976276b9c9819108adbc73c5aad636e5353e20746b73069/scipy-1.17.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee", size = 20334032, upload-time = "2026-02-23T00:16:17.358Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e6/cef1cf3557f0c54954198554a10016b6a03b2ec9e22a4e1df734936bd99c/scipy-1.17.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd", size = 22709533, upload-time = "2026-02-23T00:16:25.791Z" }, + { url = "https://files.pythonhosted.org/packages/4d/60/8804678875fc59362b0fb759ab3ecce1f09c10a735680318ac30da8cd76b/scipy-1.17.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c", size = 33062057, upload-time = "2026-02-23T00:16:36.931Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/af933f0f6e0767995b4e2d705a0665e454d1c19402aa7e895de3951ebb04/scipy-1.17.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4", size = 35349300, upload-time = "2026-02-23T00:16:49.108Z" }, + { url = "https://files.pythonhosted.org/packages/b4/3d/7ccbbdcbb54c8fdc20d3b6930137c782a163fa626f0aef920349873421ba/scipy-1.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444", size = 35127333, upload-time = "2026-02-23T00:17:01.293Z" }, + { url = "https://files.pythonhosted.org/packages/e8/19/f926cb11c42b15ba08e3a71e376d816ac08614f769b4f47e06c3580c836a/scipy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082", size = 37741314, upload-time = "2026-02-23T00:17:12.576Z" }, + { url = "https://files.pythonhosted.org/packages/95/da/0d1df507cf574b3f224ccc3d45244c9a1d732c81dcb26b1e8a766ae271a8/scipy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff", size = 36607512, upload-time = "2026-02-23T00:17:23.424Z" }, + { url = "https://files.pythonhosted.org/packages/68/7f/bdd79ceaad24b671543ffe0ef61ed8e659440eb683b66f033454dcee90eb/scipy-1.17.1-cp311-cp311-win_arm64.whl", hash = "sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d", size = 24599248, upload-time = "2026-02-23T00:17:34.561Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, + { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, + { url = "https://files.pythonhosted.org/packages/76/27/07ee1b57b65e92645f219b37148a7e7928b82e2b5dbeccecb4dff7c64f0b/scipy-1.17.1-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c", size = 31590199, upload-time = "2026-02-23T00:19:17.192Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ae/db19f8ab842e9b724bf5dbb7db29302a91f1e55bc4d04b1025d6d605a2c5/scipy-1.17.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f", size = 28154001, upload-time = "2026-02-23T00:19:22.241Z" }, + { url = "https://files.pythonhosted.org/packages/5b/58/3ce96251560107b381cbd6e8413c483bbb1228a6b919fa8652b0d4090e7f/scipy-1.17.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d", size = 20325719, upload-time = "2026-02-23T00:19:26.329Z" }, + { url = "https://files.pythonhosted.org/packages/b2/83/15087d945e0e4d48ce2377498abf5ad171ae013232ae31d06f336e64c999/scipy-1.17.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b", size = 22683595, upload-time = "2026-02-23T00:19:30.304Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e0/e58fbde4a1a594c8be8114eb4aac1a55bcd6587047efc18a61eb1f5c0d30/scipy-1.17.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6", size = 32896429, upload-time = "2026-02-23T00:19:35.536Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5f/f17563f28ff03c7b6799c50d01d5d856a1d55f2676f537ca8d28c7f627cd/scipy-1.17.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464", size = 35203952, upload-time = "2026-02-23T00:19:42.259Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a5/9afd17de24f657fdfe4df9a3f1ea049b39aef7c06000c13db1530d81ccca/scipy-1.17.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950", size = 34979063, upload-time = "2026-02-23T00:19:47.547Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/88b1d2384b424bf7c924f2038c1c409f8d88bb2a8d49d097861dd64a57b2/scipy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369", size = 37598449, upload-time = "2026-02-23T00:19:53.238Z" }, + { url = "https://files.pythonhosted.org/packages/35/e5/d6d0e51fc888f692a35134336866341c08655d92614f492c6860dc45bb2c/scipy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448", size = 36510943, upload-time = "2026-02-23T00:20:50.89Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fd/3be73c564e2a01e690e19cc618811540ba5354c67c8680dce3281123fb79/scipy-1.17.1-cp313-cp313-win_arm64.whl", hash = "sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87", size = 24545621, upload-time = "2026-02-23T00:20:55.871Z" }, + { url = "https://files.pythonhosted.org/packages/6f/6b/17787db8b8114933a66f9dcc479a8272e4b4da75fe03b0c282f7b0ade8cd/scipy-1.17.1-cp313-cp313t-macosx_10_14_x86_64.whl", hash = "sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a", size = 31936708, upload-time = "2026-02-23T00:19:58.694Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/524405c2b6392765ab1e2b722a41d5da33dc5c7b7278184a8ad29b6cb206/scipy-1.17.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0", size = 28570135, upload-time = "2026-02-23T00:20:03.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/c3/5bd7199f4ea8556c0c8e39f04ccb014ac37d1468e6cfa6a95c6b3562b76e/scipy-1.17.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce", size = 20741977, upload-time = "2026-02-23T00:20:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b8/8ccd9b766ad14c78386599708eb745f6b44f08400a5fd0ade7cf89b6fc93/scipy-1.17.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6", size = 23029601, upload-time = "2026-02-23T00:20:12.161Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a0/3cb6f4d2fb3e17428ad2880333cac878909ad1a89f678527b5328b93c1d4/scipy-1.17.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e", size = 33019667, upload-time = "2026-02-23T00:20:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/2d834a5ac7bf3a0c806ad1508efc02dda3c8c61472a56132d7894c312dea/scipy-1.17.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475", size = 35264159, upload-time = "2026-02-23T00:20:23.087Z" }, + { url = "https://files.pythonhosted.org/packages/4d/77/d3ed4becfdbd217c52062fafe35a72388d1bd82c2d0ba5ca19d6fcc93e11/scipy-1.17.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50", size = 35102771, upload-time = "2026-02-23T00:20:28.636Z" }, + { url = "https://files.pythonhosted.org/packages/bd/12/d19da97efde68ca1ee5538bb261d5d2c062f0c055575128f11a2730e3ac1/scipy-1.17.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca", size = 37665910, upload-time = "2026-02-23T00:20:34.743Z" }, + { url = "https://files.pythonhosted.org/packages/06/1c/1172a88d507a4baaf72c5a09bb6c018fe2ae0ab622e5830b703a46cc9e44/scipy-1.17.1-cp313-cp313t-win_amd64.whl", hash = "sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c", size = 36562980, upload-time = "2026-02-23T00:20:40.575Z" }, + { url = "https://files.pythonhosted.org/packages/70/b0/eb757336e5a76dfa7911f63252e3b7d1de00935d7705cf772db5b45ec238/scipy-1.17.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49", size = 24856543, upload-time = "2026-02-23T00:20:45.313Z" }, + { url = "https://files.pythonhosted.org/packages/cf/83/333afb452af6f0fd70414dc04f898647ee1423979ce02efa75c3b0f2c28e/scipy-1.17.1-cp314-cp314-macosx_10_14_x86_64.whl", hash = "sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717", size = 31584510, upload-time = "2026-02-23T00:21:01.015Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a6/d05a85fd51daeb2e4ea71d102f15b34fedca8e931af02594193ae4fd25f7/scipy-1.17.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9", size = 28170131, upload-time = "2026-02-23T00:21:05.888Z" }, + { url = "https://files.pythonhosted.org/packages/db/7b/8624a203326675d7746a254083a187398090a179335b2e4a20e2ddc46e83/scipy-1.17.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b", size = 20342032, upload-time = "2026-02-23T00:21:09.904Z" }, + { url = "https://files.pythonhosted.org/packages/c9/35/2c342897c00775d688d8ff3987aced3426858fd89d5a0e26e020b660b301/scipy-1.17.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866", size = 22678766, upload-time = "2026-02-23T00:21:14.313Z" }, + { url = "https://files.pythonhosted.org/packages/ef/f2/7cdb8eb308a1a6ae1e19f945913c82c23c0c442a462a46480ce487fdc0ac/scipy-1.17.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350", size = 32957007, upload-time = "2026-02-23T00:21:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2e/7eea398450457ecb54e18e9d10110993fa65561c4f3add5e8eccd2b9cd41/scipy-1.17.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118", size = 35221333, upload-time = "2026-02-23T00:21:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/d9/77/5b8509d03b77f093a0d52e606d3c4f79e8b06d1d38c441dacb1e26cacf46/scipy-1.17.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068", size = 35042066, upload-time = "2026-02-23T00:21:31.358Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/18f80fb99df40b4070328d5ae5c596f2f00fffb50167e31439e932f29e7d/scipy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118", size = 37612763, upload-time = "2026-02-23T00:21:37.247Z" }, + { url = "https://files.pythonhosted.org/packages/4b/39/f0e8ea762a764a9dc52aa7dabcfad51a354819de1f0d4652b6a1122424d6/scipy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19", size = 37290984, upload-time = "2026-02-23T00:22:35.023Z" }, + { url = "https://files.pythonhosted.org/packages/7c/56/fe201e3b0f93d1a8bcf75d3379affd228a63d7e2d80ab45467a74b494947/scipy-1.17.1-cp314-cp314-win_arm64.whl", hash = "sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293", size = 25192877, upload-time = "2026-02-23T00:22:39.798Z" }, + { url = "https://files.pythonhosted.org/packages/96/ad/f8c414e121f82e02d76f310f16db9899c4fcde36710329502a6b2a3c0392/scipy-1.17.1-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6", size = 31949750, upload-time = "2026-02-23T00:21:42.289Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b0/c741e8865d61b67c81e255f4f0a832846c064e426636cd7de84e74d209be/scipy-1.17.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1", size = 28585858, upload-time = "2026-02-23T00:21:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/3985219c6177866628fa7c2595bfd23f193ceebbe472c98a08824b9466ff/scipy-1.17.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39", size = 20757723, upload-time = "2026-02-23T00:21:52.039Z" }, + { url = "https://files.pythonhosted.org/packages/c0/19/2a04aa25050d656d6f7b9e7b685cc83d6957fb101665bfd9369ca6534563/scipy-1.17.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca", size = 23043098, upload-time = "2026-02-23T00:21:56.185Z" }, + { url = "https://files.pythonhosted.org/packages/86/f1/3383beb9b5d0dbddd030335bf8a8b32d4317185efe495374f134d8be6cce/scipy-1.17.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad", size = 33030397, upload-time = "2026-02-23T00:22:01.404Z" }, + { url = "https://files.pythonhosted.org/packages/41/68/8f21e8a65a5a03f25a79165ec9d2b28c00e66dc80546cf5eb803aeeff35b/scipy-1.17.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a", size = 35281163, upload-time = "2026-02-23T00:22:07.024Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/c8a5e19479554007a5632ed7529e665c315ae7492b4f946b0deb39870e39/scipy-1.17.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4", size = 35116291, upload-time = "2026-02-23T00:22:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/52/52/e57eceff0e342a1f50e274264ed47497b59e6a4e3118808ee58ddda7b74a/scipy-1.17.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2", size = 37682317, upload-time = "2026-02-23T00:22:18.513Z" }, + { url = "https://files.pythonhosted.org/packages/11/2f/b29eafe4a3fbc3d6de9662b36e028d5f039e72d345e05c250e121a230dd4/scipy-1.17.1-cp314-cp314t-win_amd64.whl", hash = "sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484", size = 37345327, upload-time = "2026-02-23T00:22:24.442Z" }, + { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, +] + +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/19/ca10ead60b0acc80b2b833c2c4a4f2ff753d0f58b811f70d911c7e94a25c/scipy-1.18.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7bd21faaf5a1a3b2eff922d02db5f191b99a6518db9078a8fb23169f6d22259a", size = 31056519, upload-time = "2026-06-19T14:59:45.203Z" }, + { url = "https://files.pythonhosted.org/packages/96/72/1e6442a00cd2924d361aa1b642ab6373ec35c6fabf311a760be9f76e0f13/scipy-1.18.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:265915e79107de9f946b855e50d7470d5893ec3f54b342e1aa6201cbdcd8bb6b", size = 28681889, upload-time = "2026-06-19T14:59:48.103Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2d/11dd93d21e147a73ba22bd75c0b9208d3a2e0ec76d53170ce7d9029b1015/scipy-1.18.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9ab7b758be6940954a713ee466e2043e9f6e2ed965c1fce5c91039f4be3d90a9", size = 20423580, upload-time = "2026-06-19T14:59:50.665Z" }, + { url = "https://files.pythonhosted.org/packages/9c/01/93552f75e0d2a7dd115a45e59209c51e8d514daff02fc887d2623be06fe1/scipy-1.18.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:97b6cddaaee0a779ef6b5ca83c9604b27cc16b2b8fc22c142652df8793319fb8", size = 23054441, upload-time = "2026-06-19T14:59:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/3c/23/21f5e703643d66f21faa6b4c73195bfcad70c55efcb4f1ab327cd7c4101a/scipy-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a96e21517c7292375c0e27dd796a811f03fcea5fd4d108fdfea8145dcf17ab", size = 33968720, upload-time = "2026-06-19T14:59:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/dd/aa/1b939f6c67ed68635bb538e6752d3dacc02f66535182e939a89581a44e9c/scipy-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f55797419e16e7f30cf88ffb3113ce0467f00cfe3f70d5c281730b21769bfc2", size = 35287115, upload-time = "2026-06-19T14:59:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ff/eec46be7e9234208f801062b53e1983085eddebd693f6c9bfb03b459830d/scipy-1.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ad033410e2e0672ffdc1042110cef20e1c46f8fd0616cee1d44d8d58fad8fc11", size = 35577989, upload-time = "2026-06-19T15:00:02.235Z" }, + { url = "https://files.pythonhosted.org/packages/84/ca/210d4759c7210bb7d269437421959b39a33434e2776b60c5cb8a763bb30a/scipy-1.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4a55985d54c769c872e64b7f4c8a81cc30ef700cc04296abbbf3705439c126de", size = 37421717, upload-time = "2026-06-19T15:00:05.102Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/9a9edb45345bd6744da5ddfb6628e5d5185920494c6a67ec45b6381004cb/scipy-1.18.0-cp312-cp312-win_amd64.whl", hash = "sha256:71ccc8faa2dd16ac310233203474a8b5cb67f10dedd54a3116d34943f4b19132", size = 36597428, upload-time = "2026-06-19T15:00:08.112Z" }, + { url = "https://files.pythonhosted.org/packages/99/0e/33f32a2a58987e26aec0f7df252cbbad1e90ae77bdbc76f40dd4ed0cf0ea/scipy-1.18.0-cp312-cp312-win_arm64.whl", hash = "sha256:d88363fd9d8fbd3511bd273f1a49efb2a540773ddf92a91d57498ce7dd7f3e76", size = 24351481, upload-time = "2026-06-19T15:00:11.103Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "snowballstemmer" version = "3.1.1" @@ -820,7 +1398,9 @@ name = "sphinx" version = "9.0.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.12'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "alabaster" }, @@ -851,8 +1431,15 @@ name = "sphinx" version = "9.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.15'", - "python_full_version >= '3.12' and python_full_version < '3.15'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ { name = "alabaster" }, @@ -999,6 +1586,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -1008,6 +1610,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" From ab625058f574ae339b508a74a1437246b966bd51 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 04:39:14 -0700 Subject: [PATCH 07/14] Pin the #272 kana-license and nakaguro behavior in the case table Co-Authored-By: Claude Fable 5 --- tests/v2/cases.py | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/v2/cases.py b/tests/v2/cases.py index e1b2a41..35e7e55 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -562,4 +562,62 @@ def __post_init__(self) -> None: "why Han segmentation is opt-in and why the gate " "cannot guard it (DEVIATES declares all Han); " "Japanese is #272's pluggable segmenter"), + + # -- #272: the kana license + nakaguro (amendment 2026-07-29). + # Segmenter-dependent divisions cannot live here (Case has no + # segmenter field); they are pinned in test_locales.py's + # integration tests instead. + Case("ja_kana_spaced_family_first", "高橋 みなみ", + {"family": "高橋", "given": "みなみ"}, + classification="fix(#272)", + notes="hiragana identifies Japanese as certainly as hangul " + "identifies Korean; kana-licensed names read " + "family-first by default"), + Case("ja_kanji_katakana_pieces", "山田 エミ", + {"family": "山田", "given": "エミ"}, + classification="fix(#272)", + notes="a kanji piece + a katakana piece cannot be a foreign " + "transcription (those are katakana-only): native, " + "licensed"), + Case("ja_unspaced_unsegmented_default", "高橋みなみ", + {"family": "高橋みなみ"}, + classification="fix(#272)", + notes="no segmenter by default: one token, family role via " + "the kana license"), + Case("ja_pure_katakana_positional", "マイケル ジャクソン", + {"given": "マイケル", "family": "ジャクソン"}, + notes="parity row guarding the license's boundary: " + "pure-katakana is predominantly transcribed foreign " + "names in original order -- never licensed"), + Case("ja_nakaguro_divides_the_transcription", "マイケル・ジャクソン", + {"given": "マイケル", "family": "ジャクソン"}, + classification="fix(#272)", + notes="the katakana middle dot is the transcription's own " + "part divider: it separates like whitespace, the " + "license declines each katakana token, and the " + "positional default keeps the source-language order"), + Case("ja_nakaguro_han_takes_the_han_order", "高橋・一郎", + {"family": "高橋", "given": "一郎"}, + classification="fix(#272)", + notes="the dot splits; both tokens are pure Han, so the HAN " + "family-first entry fires -- the katakana row's sibling " + "through the other outcome"), + Case("ja_lone_hiragana_takes_family", "みなみ", + {"family": "みなみ"}, + classification="fix(#272)", + notes="hiragana earns a script_orders entry in its own right: " + "a lone token takes the entry's first role"), + Case("ja_lone_katakana_stays_given", "マイケル", + {"given": "マイケル"}, + notes="parity: katakana deliberately has no entry, so the " + "positional default holds -- transcribed foreign names " + "keep source order"), + Case("ja_nakaguro_inside_a_nickname", + "山田 太郎 (マイケル・ジャクソン)", + {"family": "山田", "given": "太郎", + "nickname": "マイケル ジャクソン"}, + classification="fix(#272)", + notes="delimited content tokenizes under the same separator " + "rules: the dot renders back as a space in the nickname " + "join -- a decision, not an accident"), ) From d3e7596471842bfc1ef04228d66175c129121bb0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 04:56:04 -0700 Subject: [PATCH 08/14] Extend the guard rails to the #272 fields The property fuzzer's script_orders sample and several source docstrings still described the pre-#272 world: four script members where the comment said two, and three forward references to "#272's pluggable segmenter" pointing at code that has now landed in this branch. _SCRIPT_ORDER_TABLES gains a table keyed on Script.HIRAGANA, the kana license's carrier -- the one key reached indirectly, since a mixed kanji+kana name resolves to it rather than to any script it is literally written in, and pointing it at GIVEN_FIRST makes a consulted table distinguishable from the default it would otherwise agree with. AGENTS.md's scoped-exception paragraph gains the kana license itself (hiragana identifies Japanese; pure katakana is excluded because transcribed foreign names keep their source order), the corrected count, and the pack-layering line's new _types entry. test_regex_sync.py's kana work -- the differential-toml pin extension, the _SCRIPT_RANGES completeness assertion and the docstring's stale scoping premise -- lands with the toml change it pins, so that each commit is green on its own. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- nameparser/_pipeline/_script_segment.py | 3 ++- nameparser/_policy.py | 19 +++++++++++++------ nameparser/locales/zh.py | 5 ++++- tests/v2/cases.py | 3 ++- tests/v2/test_properties.py | 24 +++++++++++++++--------- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 07e7530..8ca3c5a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Three defaults fall out of it. Wholly-Han and wholly-Hangul names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Han segmentation stays OPT-IN (`locales.ZH`) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎 (Japanese needs #272's pluggable segmenter). Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Four defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) @@ -118,7 +118,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a checked-in corpus of ~650 names (no exact count here: `corpus_issues.jsonl` grows whenever it is regenerated, and the run prints its own per-file totals) (two files: `corpus.jsonl` from the v1 test banks at a pinned ref, `corpus_issues.jsonl` harvested from the issue tracker; `compare.py` globs `corpus*.jsonl` and fails loudly if none match). `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy` only, and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 6288b0b..454323c 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -58,7 +58,8 @@ unambiguously Korean, and its surname set is closed and default-shipped), while HAN is opt-in via locales.ZH -- a Chinese surname list corrupts Japanese names ("高橋一郎" must not split as -高 + 橋一郎), which is #272's pluggable segmenter, not this table's. +高 + 橋一郎). Japanese divides through the segmenter path below under +locales.JA, not through this table's vocabulary. The gate resolves each token through effective_script, the same function order resolution uses, so kana-licensed composites (高橋みなみ -> HIRAGANA) gate in under the JA pack's HIRAGANA entry while diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 42ed2d8..78a1fb5 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -41,7 +41,9 @@ class Script(StrEnum): #: Chinese Hanzi -- and Japanese Kanji: a pure-Han string cannot #: say which language it is, which is fine for ORDER (both write #: family-first natively) and exactly why Han SEGMENTATION is - #: opt-in (locales.ZH; Japanese is #272's pluggable segmenter). + #: opt-in, per language: locales.ZH brings the Chinese surname + #: list, locales.JA activates the same stage for a pluggable + #: segmenter to divide kanji names with. HAN = "han" #: Korean Hangul (precomposed syllables). Unambiguously Korean. HANGUL = "hangul" @@ -432,11 +434,16 @@ class Policy: #: Scripts for which the unspaced-name segmentation stage is #: active (#271): the first token written wholly in an activated #: script is split by longest surname match against - #: :attr:`Lexicon.surnames `. - #: Default: {Script.HANGUL} -- hangul is unambiguously Korean and - #: Korean surnames are a closed default-shipped set. Han is NOT - #: default: a zh surname list corrupts Japanese names (高橋一郎 - #: must not split as 高+橋一郎), so it's opt-in via locales.ZH. + #: :attr:`Lexicon.surnames `, and, + #: where that vocabulary declines, by a + #: :data:`~nameparser.Segmenter` if one was given to the parser + #: (#272). Default: {Script.HANGUL} -- hangul is unambiguously + #: Korean and Korean surnames are a closed default-shipped set. + #: Han is NOT default: a zh surname list corrupts Japanese names + #: (高橋一郎 must not split as 高+橋一郎), so it's opt-in via + #: locales.ZH for Chinese and locales.JA -- which activates + #: Script.HIRAGANA alongside it, the kana license's carrier key -- + #: for Japanese. #: Opt out with ``segment_scripts=()``; note a PolicyPatch unions #: rather than replaces, so a pack can only add scripts, never #: disable one. diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py index 00e1f23..30d8e06 100644 --- a/nameparser/locales/zh.py +++ b/nameparser/locales/zh.py @@ -6,7 +6,10 @@ cannot say whether it is Chinese or Japanese (林 is Lin or Hayashi) and a zh surname list corrupts Japanese names (高橋一郎 would split 高 + 橋一郎). Applying this pack IS the "my data is Chinese" -declaration. Japanese is #272's pluggable segmenter. +declaration. Japanese segmentation is locales.JA's job instead: that +pack activates the same stage for the Japanese repertoire and defers +the division itself to a pluggable segmenter, since no surname list +divides a kanji name. Data sources: single-character surnames are the top-100 of the 2020 PRC census; traditional-script variant forms are included for names diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 35e7e55..f0a48bf 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -561,7 +561,8 @@ def __post_init__(self) -> None: "names mis-split (高橋 is the real surname). This is " "why Han segmentation is opt-in and why the gate " "cannot guard it (DEVIATES declares all Han); " - "Japanese is #272's pluggable segmenter"), + "Japanese data belongs under locales.JA and its " + "segmenter instead"), # -- #272: the kana license + nakaguro (amendment 2026-07-29). # Segmenter-dependent divisions cannot live here (Case has no diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index c17c694..e66b4b4 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -245,17 +245,23 @@ def _lexicons(draw: st.DrawFn) -> Lexicon: # script_orders' legal values are as restricted as name_order's (only # the three exported orders, keyed by Script), so they are sampled -# rather than generated. The four cover the axes that matter: the -# shipped default, the full opt-out, one script alone, and an order -# that disagrees with the default -- FAMILY_FIRST_GIVEN_LAST on hangul +# rather than generated. The five cover the axes that matter: the +# shipped default, the full opt-out, one script alone, an order that +# disagrees with the default -- FAMILY_FIRST_GIVEN_LAST on hangul # reads "김민준 수" differently from every other entry here, which is # what makes a script table that is merely PRESENT distinguishable -# from one that is actually consulted. +# from one that is actually consulted -- and a table keyed on the kana +# license's carrier (#272). That last row is the only one whose key is +# reached INDIRECTLY: a mixed kanji+kana name resolves to HIRAGANA +# rather than to any script it is literally written in, and pointing +# the carrier back at GIVEN_FIRST makes a consulted table visibly +# different from the default it would otherwise agree with. _SCRIPT_ORDER_TABLES = [ DEFAULT_SCRIPT_ORDERS, (), ((Script.HAN, FAMILY_FIRST),), ((Script.HANGUL, FAMILY_FIRST_GIVEN_LAST),), + ((Script.HIRAGANA, GIVEN_FIRST),), ] @@ -271,11 +277,11 @@ def _policies(draw: st.DrawFn) -> Policy: name_order=draw(st.sampled_from( [GIVEN_FIRST, FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST])), script_orders=draw(st.sampled_from(_SCRIPT_ORDER_TABLES)), - # no max_size: Script has two members, so the unbounded draw - # already reaches every subset -- including the empty one, - # which is the documented segmentation opt-out, and the full - # one, which turns on the Han segmentation that only - # locales.ZH turns on in shipped configuration + # no max_size: Script has four members, so the unbounded draw + # still reaches every subset -- including the empty one, which + # is the documented segmentation opt-out, and the full one, + # which turns on the Han and kana activation that only + # locales.ZH and locales.JA turn on in shipped configuration segment_scripts=draw(st.frozensets(st.sampled_from(list(Script)))), patronymic_rules=draw(st.frozensets( st.sampled_from(list(PatronymicRule)), max_size=2)), From 37cf7589fd95764f32d491c4ddfa3f6c66733046 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 05:03:06 -0700 Subject: [PATCH 09/14] Document the Japanese support (#272) usage.rst gains the Japanese story in the East Asian section, told background first: how a Japanese name is written across kanji and the two kana syllabaries, why hiragana identifies a native name and katakana does not, then the two behaviors that follow without configuration (kana-licensed family-first order, the nakaguro as a token separator), then the segmenter opt-in, then the boundaries -- pure katakana, romanized text, the two-character rule, the SEGMENTATION report's own wording, silent declines, and the CLI's inability to attach a segmenter. locales.rst gains a Segmenters section carrying the decline contract (segment_scripts unions, so a segmenter is offered every activated script's tokens and must decline what it cannot read) and the empty-lexicon pack recipe ja is the shipped example of; customize.rst states that the kana behaviors ride the two existing switches and that the middle dot rides neither; concepts.rst qualifies "parsing never raises" with the one exception a user-supplied callable creates. migrate.rst's CJK section absorbs the new shapes rather than growing a parallel one, every claim checked shape by shape against the pinned 1.4 worker: the spaced flips, the lone-token moves, both nakaguro outcomes, the nickname render, and the katakana non-change. Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 6 +- docs/customize.rst | 24 ++++++-- docs/locales.rst | 76 +++++++++++++++++++----- docs/migrate.rst | 62 ++++++++++++++++++-- docs/modules.rst | 30 +++++++++- docs/release_log.rst | 10 +++- docs/usage.rst | 136 +++++++++++++++++++++++++++++++++++++------ 7 files changed, 297 insertions(+), 47 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index dda5d82..25b612b 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -146,7 +146,11 @@ Honest ambiguity Parsing never raises. Pass in a string that doesn't look like a name at all, and you get back a :class:`~nameparser.ParsedName` with empty fields, not an exception. The parser's job is to make a reasonable -call on real-world text, not to reject it. +call on real-world text, not to reject it. The single exception is +code you supplied yourself: a :data:`~nameparser.Segmenter` passed to +``Parser(segmenter=...)`` runs inside the parse, and its own +exceptions propagate rather than being swallowed — a failure there is +a bug in your callable, not a fact about the name. Some calls are irreducibly ambiguous — both readings are legitimate, and no amount of rule-tuning resolves them without breaking some other diff --git a/docs/customize.rst b/docs/customize.rst index 126fa40..2f29d70 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -307,12 +307,12 @@ East Asian defaults, and turning them off ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Two defaults key on the *script* a name is written in rather than on -anything you set: a name written wholly in Han or Hangul is assigned -family-first (``script_orders``), and an unspaced hangul name is split -into surname and given name against the shipped Korean census list -(``segment_scripts``). :ref:`east-asian-names` explains the naming -conventions both rest on — this section is how to switch them off, -which you can do separately: +anything you set: a name written wholly in Han or Hangul — or one +mixing kanji with kana — is assigned family-first (``script_orders``), +and an unspaced hangul name is split into surname and given name +against the shipped Korean census list (``segment_scripts``). +:ref:`east-asian-names` explains the naming conventions both rest on — +this section is how to switch them off, which you can do separately: .. doctest:: @@ -345,6 +345,18 @@ Chinese surnames are deliberately absent from that default set, because splitting Han text requires knowing Chinese from Japanese; :doc:`locales` covers the opt-in ``zh`` pack that supplies them. +The Japanese behaviors ride these same two fields, so they need no +switches of their own: ``script_orders={}`` clears the kana-licensed +entry along with the Han and Hangul ones, and ``segment_scripts=()`` +deactivates every script at once, which also stops a parser consulting +whatever segmenter it was given. The segmenter has an off-switch as +well — ``Parser(segmenter=None)``, which is the default; see +:ref:`segmenter-contract` for what one is expected to do with text it +does not handle. One Japanese behavior is not a policy field at all: +the katakana middle dot ・ separates tokens the way a space does, +decided in tokenization, so it applies however these two fields are +set. + .. note:: Both fields are annotated with their canonical *storage* type diff --git a/docs/locales.rst b/docs/locales.rst index 0bd9f87..300a267 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -61,17 +61,18 @@ background (covered fully under :ref:`east-asian-names` in :doc:`usage`): Chinese, Japanese, and Korean names put the family name first in native script and are usually written with no space between the parts. Both defaults follow from facts the script alone -establishes. A name written wholly in Han or Hangul is assigned +establishes. A name written wholly in Han or Hangul — or one mixing +kanji with kana, a combination only Japanese produces — is assigned family-first, because every language written in those scripts orders -names that way — no language guess is involved. An unspaced hangul +names that way; no language guess is involved. An unspaced hangul name is additionally split into surname and given name, because hangul is written by nothing but Korean and Korean surnames are a closed census set that ships as default vocabulary. Splitting an unspaced *Han* name is the one behavior the script cannot license — the same characters could be a Chinese or a Japanese name, and a Chinese surname list splits Japanese names in the wrong place — so it is not a -default: the ``zh`` pack below turns it on when you can declare the -data Chinese. +default: the ``zh`` and ``ja`` packs below turn it on for data whose +language you can declare. A pack is for something different: a *structural* rule, like reordering a patronymic, that vocabulary alone can't express. @@ -175,6 +176,47 @@ new naming rule belongs in. mixes traditions, parse the subsets separately with different parsers rather than enabling a pack over all of it. +.. _segmenter-contract: + +Segmenters +----------- + +``ja`` is policy-only in a second sense: it turns division on for +Japanese text without supplying anything to divide with. That job goes +to a **segmenter**, which is passed to +:func:`~nameparser.parser_for` rather than carried by the pack — a +:class:`~nameparser.Locale` is pure data, and a third-party callable is +neither pure nor data. :func:`~nameparser.locales.ja_segmenter` is the +shipped one; writing your own is worth it for any script whose +divisions you know better than a surname list does. + +A :data:`~nameparser.Segmenter` is any callable taking a token's text +and returning a :class:`~nameparser.Segmentation` — the interior +offsets to cut at, plus how confident you are — or ``None`` to decline, +leaving the token whole. Declining is the load-bearing half of the +contract, because ``segment_scripts`` unions across packs: your +segmenter is offered every token of every activated script, not only +the ones its own pack turned on. Recognize the text you can actually +read and return ``None`` for the rest, rather than answering for a +script you never meant to handle. Exceptions are the one thing that +does not stay inside the parse — a segmenter is your code, so its +errors propagate out of ``parse()`` instead of being absorbed as +content errors. + +The pack half of that arrangement carries no words at all, which makes +it the shortest kind of pack there is: + +.. doctest:: + + >>> from nameparser import Lexicon, Locale, PolicyPatch, Script + >>> mine = Locale(code="myscript", lexicon=Lexicon.empty(), + ... policy=PolicyPatch( + ... segment_scripts=frozenset({Script.HAN}))) + +``nameparser/locales/ja.py`` is the shipped example of exactly that +shape: activation is the pack's entire contribution, and a pack +applied without a segmenter simply divides nothing. + Creating your own Locale ------------------------- @@ -245,11 +287,14 @@ by ``tests/v2/test_locales.py``: needs at least one name exercising every alternation branch of every regex it defines — ``test_rotators_cover_every_marker_branch`` fails until each branch is hit. A pack declaring by *codepoint range* - (``zh``) has no branches to sweep and drops out of that test, so its - rotators have to carry the same weight by hand: the unspaced names - the pack must split, one per shape of the vocabulary it ships — - single surname, compound surname, and any spelling variant it means - to cover. + (``zh``, ``ja``) has no branches to sweep and drops out of that + test, so its rotators have to carry the same weight by hand: the + unspaced names the pack must split, one per shape of the vocabulary + it ships — single surname, compound surname, and any spelling + variant it means to cover. A pack that ships no vocabulary lists + the shapes its *segmenter* must divide instead, and marks the + rotator tests to skip when the optional dependency is absent, so + the contract tests still run everywhere. #. Keep the non-interference gate green over the shared corpus plus your rotators: every name the packed parser parses differently from the default must be one your ``DEVIATES`` predicate flags — no @@ -264,16 +309,17 @@ by ``tests/v2/test_locales.py``: language its script does not* — a Chinese surname list, which silently mangles the Japanese names written in the same characters — belongs in the pack, where asking for it is the declaration. - ``ru`` and ``tr_az`` need no vocabulary at all and ship an empty - :class:`~nameparser.Lexicon`; ``nameparser/locales/zh.py`` is the - template for one that does. + ``ja``, ``ru`` and ``tr_az`` need no vocabulary at all and ship an + empty :class:`~nameparser.Lexicon`; ``nameparser/locales/zh.py`` is + the template for one that does. #. Curate vocabulary conservatively, the same rule as :doc:`customize`: when you're unsure whether a word or a marker belongs, leave it out. ``nameparser/locales/ru.py`` is the reference implementation for a policy-only pack, ``nameparser/locales/zh.py`` for one that carries -vocabulary. Packs still in progress are tracked in issues `#272 -`_ (Japanese) -and `#146 `_ +vocabulary, and ``nameparser/locales/ja.py`` for one whose whole +contribution is turning a stage on. Packs still in progress are +tracked in issue `#146 +`_ (Vietnamese). diff --git a/docs/migrate.rst b/docs/migrate.rst index 9760b37..3ede387 100644 --- a/docs/migrate.rst +++ b/docs/migrate.rst @@ -340,10 +340,11 @@ custom suffix delimiter configured, a no-space delimiter group renders whole (``"RN/CRNA"``) where 1.x split it (``"RN, CRNA"``) — the role assignment is identical, only the rendered string differs. -2.1 adds two more, and unlike most of the 2.0 API these do reach -``HumanName``: a name written wholly in Han or Hangul is read -family-first, and an unspaced Korean name is split into surname and -given name. +2.1 adds three more, and unlike most of the 2.0 API these do reach +``HumanName``: a name written in East Asian script is read +family-first, an unspaced Korean name is split into surname and given +name, and the katakana middle dot separates tokens the way a space +does. .. doctest:: @@ -377,7 +378,58 @@ Both were ``first`` under 1.4. If you feed unspaced CJK through reach you, and it is silent — the string is intact, just in the other field. +Japanese kana carries the same order rule, which widens both shapes +past the wholly-Han text described above. A name that mixes kanji with +hiragana or katakana is read family-first, and a lone kana-bearing +token moves from ``first`` to ``last`` exactly as ``毛泽东`` does: + +.. doctest:: + + >>> HumanName("高橋 みなみ").last, HumanName("高橋 みなみ").first + ('高橋', 'みなみ') + >>> HumanName("山田 エミ").last + '山田' + >>> HumanName("高橋みなみ").last, HumanName("高橋みなみ").first + ('高橋みなみ', '') + +1.4 read the spaced ones as ``first="高橋"``/``last="みなみ"`` and +``first="山田"``/``last="エミ"``, and put the unspaced one whole in +``first``. The spaced shapes change what the name renders as too: +``str(HumanName("高橋 みなみ"))`` was ``"高橋 みなみ"`` and is now +``"みなみ 高橋"``. A name written *wholly* in katakana is deliberately +left alone — it is usually a transcribed foreign name already in +given-first order — so ``HumanName("マイケル ジャクソン")`` reads +``first="マイケル"``/``last="ジャクソン"`` on both versions. + +One more shape changes for a different reason: the katakana middle dot +``・``, which divides the parts of such a transcription, is now a token +separator rather than an ordinary character. 1.4 saw one token and put +it in ``first``; 2.1 sees two: + +.. doctest:: + + >>> HumanName("マイケル・ジャクソン").first + 'マイケル' + >>> HumanName("マイケル・ジャクソン").last + 'ジャクソン' + >>> HumanName("高橋・一郎").last, HumanName("高橋・一郎").first + ('高橋', '一郎') + +The two divide the same way and land in opposite fields, because the +katakana pair keeps its source order while the kanji pair takes the +family-first rule. Separating also changes the rendered string, the +same way the Korean split does: the dot comes back as a space, so +``str(HumanName("マイケル・ジャクソン"))`` was ``"マイケル・ジャクソン"`` +and is now ``"マイケル ジャクソン"``. That reaches delimited content +too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` was +``"マイケル・ジャクソン"`` under 1.4 and is ``"マイケル ジャクソン"`` +now. + ``Constants`` has no switch for any of this — the v1 configuration surface is frozen for 2.x — so the way out is the 2.0 API: ``Parser(policy=Policy(script_orders={}, segment_scripts=()))`` -restores 1.4's reading of all three shapes. +restores 1.4's reading of every shape above that turns on order or +splitting. The middle dot is the one exception: it is decided in +tokenization rather than by policy, so a name written with one still +divides at the dot, and still renders with a space, whatever those two +fields are set to. diff --git a/docs/modules.rst b/docs/modules.rst index 8c8e7f6..7f50dee 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -14,6 +14,25 @@ Parsing .. autofunction:: nameparser.parser_for +.. autoclass:: nameparser.Segmentation + :members: + +.. py:data:: nameparser.Segmenter + :value: Callable[[str], Segmentation | None] + + The type of the optional ``Parser(segmenter=...)`` hook: a callable + given one token's text, returning a + :class:`~nameparser.Segmentation` that divides it, or ``None`` to + decline and leave it whole. An alias, not a class — any callable of + that shape qualifies, and nothing needs to be subclassed or + registered. It is consulted only for tokens whose script is listed + in :attr:`Policy.segment_scripts + `, and only where the surname + vocabulary declined first; see :ref:`segmenter-contract` for what a + segmenter owes its caller. + :func:`~nameparser.locales.ja_segmenter` is the shipped + implementation. + Results ~~~~~~~ @@ -106,10 +125,15 @@ because only these three orders have defined assignment semantics. e.g. Vietnamese full-name order. .. py:data:: nameparser.DEFAULT_SCRIPT_ORDERS - :value: ((Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST)) + :value: ((Script.HAN, FAMILY_FIRST), (Script.HANGUL, FAMILY_FIRST), (Script.HIRAGANA, FAMILY_FIRST)) The default :attr:`~nameparser.Policy.script_orders` table: a name - written wholly in Han or Hangul reads family-first. A matching + written wholly in Han or Hangul reads family-first, and so does a + Japanese name mixing kanji with kana, which resolves to the + ``HIRAGANA`` entry whichever of the two syllabaries it actually + uses — that member is the license's carrier key, not a claim about + the characters present. A name written wholly in katakana has no + entry on purpose and stays positional. A matching entry in this table takes precedence over ``name_order``, including a ``name_order`` you set explicitly — ``name_order`` governs only the names no entry matches. The values are drawn from the same @@ -147,6 +171,8 @@ Locales .. automodule:: nameparser.locales :members: get, available +.. autofunction:: nameparser.locales.ja_segmenter + 1.x compatibility layer ------------------------ diff --git a/docs/release_log.rst b/docs/release_log.rst index 95b9a35..0a82fd0 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -4,18 +4,26 @@ Release Log **East Asian name support** - - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese needs the pluggable segmenter staged in #272. It sets no name order, since native-script Han already reads family-first without it. ``locales.available()`` is now ``('ru', 'tr_az', 'zh')`` (#271) + - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese data goes through ``locales.JA`` and its segmenter instead. It sets no name order, since native-script Han already reads family-first without it. ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) + - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use (closes #272) + - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest (#272) + - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) **Breaking Changes** - Change the pickle compatibility of the 2.0 API's ``Policy`` and ``Lexicon``: the new fields change the field layout their guarded ``__setstate__`` checks, so a pickle written by 2.0.0 raises ``ValueError`` naming the missing fields instead of loading. Re-pickle after upgrading. ``HumanName`` pickles are unaffected — the facade pickles v1-shaped component state, not these objects + - Change the pickle compatibility of ``Parser`` in the same way and for the same reason: the new ``segmenter`` field changes the layout its guarded ``__setstate__`` checks, so a ``Parser`` pickled by 2.0.x raises ``ValueError`` naming the missing field rather than loading silently with the field absent. That guard is the design, not a regression — re-pickle after upgrading. A ``Parser`` carrying a segmenter pickles only if that segmenter does, which a module-level function does and a closure or lambda does not (#272) + - Change one thing about parse totality: ``parse()`` still never raises on any input, but a user-supplied ``Parser(segmenter=...)`` runs inside the parse and its own exceptions propagate rather than being absorbed. A failure there is a bug in your callable, not a fact about the name, and hiding it would only make it harder to find (#272) **Behavior Changes** - Fix names written wholly in Han or Hangul parsing given-first: native-script CJK now reads family-first by default, through the new ``Policy.script_orders`` table, so ``"毛 泽东"`` gives family ``毛`` where 1.x gave family ``泽东``. A name that is a single unspaced token moves the same way — ``"毛泽东"`` and ``"山田太郎"`` now land in ``family``/``last`` where 1.x put them in ``given``/``first``, with the string itself untouched, which makes it the easiest form of this change to miss — a lone token renders the same whichever field holds it, whereas the spaced form does show up in the output (``str(HumanName("毛 泽东"))`` is now ``"泽东 毛"``). No language detection is involved: Chinese and Japanese both write the family name first in native script, so the script settles the order without anyone having to know which language it is. Latin-script and mixed-script names are never affected, and an explicit comma still wins. **Default-on: changes parse output for wholly-CJK names**, through ``HumanName`` as well as the 2.0 API (closes #271) - Fix unspaced Korean names not splitting: the census surname list now ships as default vocabulary (``Lexicon.surnames``) with hangul segmentation on by default (``Policy.segment_scripts``), so ``"김민준"`` parses family ``김``, given ``민준`` where 1.x returned the whole string as ``first``. Rendering follows the split, so ``str(HumanName("김민준"))`` is now ``"민준 김"`` where 1.x echoed the input back unchanged. Nothing but Korean is written in hangul and its surnames are a closed census set, which is what makes the split safe as a default rather than a pack. **Default-on**, and it reaches ``HumanName`` too. ``Policy(segment_scripts=())`` turns the split off; ``Policy(script_orders={})`` separately restores the positional reading; clearing both restores 2.0 behavior exactly (#271) + - Fix Japanese names carrying kana parsing given-first: the family-first rule above extends to any name whose characters stay within kanji and kana while carrying at least one kana character, so ``"高橋 みなみ"`` gives family ``高橋`` and ``"山田 エミ"`` family ``山田`` where 1.x read both the other way round, and a lone such token (``"高橋みなみ"``, ``"みなみ"``) lands in ``family``/``last`` where 1.x put it in ``given``/``first``. The reasoning is the one hangul already uses: hiragana never transcribes a foreign name, and a transcription is kana ALONE, so kanji-plus-kana is a Japanese person's name written in Japanese order. A name written **wholly in katakana** is deliberately excluded and stays positional — it is predominantly a transcribed foreign name (``"マイケル ジャクソン"``) already in given-first order. **Default-on: changes parse output for kana-bearing Japanese names**, through ``HumanName`` as well as the 2.0 API, and the spaced forms change what the name renders as (``str(HumanName("高橋 みなみ"))`` is now ``"みなみ 高橋"``). ``Policy(script_orders={})`` clears this entry along with the Han and Hangul ones (#272) + - Fix the katakana middle dot ``・`` (U+30FB, and its halfwidth twin U+FF65) being read as part of a name rather than as the divider it is: it now separates tokens exactly as a space does. A transcribed foreign name therefore divides into its parts and, being wholly katakana, keeps its source order — ``"マイケル・ジャクソン"`` gives given ``マイケル``, family ``ジャクソン``, where 1.x left the whole string in ``first`` — while a kanji pair written the same way takes the family-first rule (``"高橋・一郎"`` → family ``高橋``). Native Japanese names never contain this character and no other script's names use it, so the separation is unconditional, which also means it is **not** covered by the two policy opt-outs. Rendering follows, the way the Korean split's does: the dot comes back as a space, so ``str(HumanName("マイケル・ジャクソン"))`` is now ``"マイケル ジャクソン"``, and that reaches delimited content too — the nickname in ``"山田 太郎 (マイケル・ジャクソン)"`` renders ``"マイケル ジャクソン"``. The Chinese interpunct ``·`` is deliberately NOT included: U+00B7 is also the Catalan punt volat and appears inside legitimate names (``Gal·la``) (#272) + - Fix NFD-decomposed input missing the East Asian defaults entirely: script classification now normalizes to NFC before deciding, so a Korean or Japanese name typed on macOS — where decomposed text is routine — gets the same order rule as its composed twin, which it silently did not before. Segmentation MATCHING deliberately stays raw, so an unspaced NFD hangul name is ordered correctly but not split, rather than being split in the wrong place. One gotcha worth stating plainly: parse output preserves the encoding it was given, so for NFD input ``name.family == "김"`` is ``False`` even though it is the same name — compare NFC-normalized text when comparing across encodings (#272) * 2.0.0 - July 27, 2026 diff --git a/docs/usage.rst b/docs/usage.rst index 7eed066..a133f00 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -180,11 +180,7 @@ does not identify the language: 高橋一郎 is a Japanese name whose family name is 高橋, but 高 alone is a common Chinese surname, so a Chinese surname list would split it in the wrong place. Declaring the language is up to you. When you know the data is Chinese, apply the -``zh`` locale pack; Japanese needs a dictionary-backed segmenter and -is tracked as `#272 -`_ — until -then an unspaced Japanese name stays whole, in the family field per -the assignment rule above: +``zh`` locale pack, which carries the surname list the split needs: .. doctest:: @@ -192,18 +188,124 @@ the assignment rule above: >>> parser_for(locales.ZH).parse("毛泽东").family '毛' -Three boundaries on all of the above. Romanized names ("Kim Min-jun") -are Latin script and follow the ordinary positional rules — order -genuinely varies in romanized data, so nothing script-based applies. -A comma disables both behaviors, on the reasoning ``name_order`` -already follows: whoever wrote the comma has already said where the -family name ends. And when an unspaced name has more than one -vocabulary-supported split — ``남궁민수`` is 남궁 + 민수 by the -two-syllable surname but 남 + 궁민수 by the single-syllable one — the -longest surname wins and the parse records the decision as an -``AmbiguityKind.SEGMENTATION``, described under `When the parser had -to guess`_; a name with only one possible split reports nothing. -:doc:`customize` covers turning either behavior off. +Japanese +~~~~~~~~~ + +Japanese writes a name in three scripts at once. Family names and most +given names are kanji — the same characters Chinese uses — but a given +name is often written in one of the two kana syllabaries instead: +hiragana (高橋みなみ) or katakana (山田エミ). The two syllabaries carry +different information about whose name it is. Hiragana never +transcribes a foreign name, so a name that mixes kanji and kana is a +Japanese person's name, written in Japanese order. Katakana is +ambiguous: native given names use it, but katakana is also how +Japanese text writes a *foreign* name — マイケル・ジャクソン is Michael +Jackson — and a transcription keeps the source language's order, given +name first, its parts divided by the middle dot ・ (the nakaguro, +U+30FB) rather than by a space. + +Two behaviors follow from that without any configuration. A name whose +characters stay within kanji and kana and carry at least one kana is +assigned family-first, like any other native-script East Asian name: it +cannot be Chinese, and it is not a transcription, because a +transcription would have been kana alone. + +.. doctest:: + + >>> minami = parse("高橋 みなみ") + >>> minami.family, minami.given + ('高橋', 'みなみ') + >>> parse("山田 エミ").family + '山田' + +And the middle dot separates tokens the way a space does, so a +transcribed foreign name divides into its parts — which, being wholly +katakana, keep the order they were written in: + +.. doctest:: + + >>> michael = parse("マイケル・ジャクソン") + >>> michael.given, michael.family + ('マイケル', 'ジャクソン') + +Dividing an *unspaced* Japanese name is a separate matter, and one no +surname list can settle: family and given names draw on the same +kanji, both sides run one to four characters, and the reading rather +than the spelling decides most divisions. nameparser therefore takes a +**segmenter** — a callable from token text to a division — and ships a +factory wrapping `namedivider-python +`_, an optional +dependency installed with the ``ja`` extra: + +.. code-block:: console + + $ pip install "nameparser[ja]" + +.. code-block:: python + + from nameparser import locales, parser_for + + parser = parser_for(locales.JA, segmenter=locales.ja_segmenter()) + parser.parse("山田太郎").family # '山田' + +Both halves are required, and they do different jobs: the ``ja`` pack +activates division for Japanese text, and the segmenter performs it. +``ja_segmenter()`` wraps namedivider's ``BasicNameDivider``, which +reads data bundled in the installed package; +``ja_segmenter(gbdt=True)`` selects its gradient-boosted divider +instead, which is more accurate and downloads its model and surname +files from the network on first use — worth knowing before deploying +it somewhere sandboxed or air-gapped. + +Boundaries +~~~~~~~~~~~ + +Several boundaries apply to all of the above. Romanized names ("Kim +Min-jun", "Yamada Taro") are Latin script and follow the ordinary +positional rules — order genuinely varies in romanized data, so +nothing script-based applies. A name written wholly in katakana stays +positional for the reason given above, pack or no pack: it is +predominantly a transcription, and a transcription is already in the +order it should be read in. And a comma disables the script behaviors +entirely, on the reasoning ``name_order`` already follows: whoever +wrote the comma has already said where the family name ends. + +A division the parser had to choose is reported rather than hidden. +When an unspaced name has more than one vocabulary-supported split — +``남궁민수`` is 남궁 + 민수 by the two-syllable surname but 남 + 궁민수 +by the single-syllable one — the longest surname wins and the parse +records the decision as an ``AmbiguityKind.SEGMENTATION``, described +under `When the parser had to guess`_; a name with only one possible +split reports nothing. A segmenter's answer is reported on the same +kind whenever its confidence falls below the stage's floor, naming the +division and the score in the report's ``detail``:: + + "'山田太郎' splits as '山田' + '太郎' on a segmenter answer scoring 0.44, under the 0.9 confidence floor" + +With namedivider that line separates its two kinds of answer. A +division it states as a rule — the kanji-to-kana boundary in 高橋みなみ +— scores 1.0 and reports nothing; a division read off kanji statistics +scores far below the floor and always reports. Read the report as a +statement about the *kind* of answer, not as a measure of how likely +this particular one is to be wrong. + +A lone two-character kanji name divides one character to each side, on +namedivider's rule for that length. A name that short carries no +evidence of where its own boundary falls, and one character each way is +the presumption Japanese practice makes; that is an accepted +presumption, not a measurement, so a two-character token is the shape +to check first if a division looks wrong. + +A segmenter that answers outside the token it was given — a cut past +the end of the text, or pieces that do not reconstruct the input — is +declined silently and the token is left whole. Exceptions are the one +thing that does *not* stay inside the parse: a segmenter is your code, +so its errors propagate rather than being absorbed as content errors. + +The command line takes the pack but not the segmenter: ``python -m +nameparser --locale ja`` has no way to attach one, so it activates +nothing by itself. :doc:`customize` covers turning any of these +behaviors off. Aggregate views ---------------- From 5f5bb5e0cf1b9f338ecf84722e581b4b9500304e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 05:05:21 -0700 Subject: [PATCH 10/14] Classify the #272 kana fixes in the differential harness The CJK rule's character class gains the two kana blocks, and its label names both issues: #272's kana license and nakaguro separator move name pieces between the same first/middle/last fields on the same native-script input, so this is one diff shape rather than two, and splitting the rule by issue would buy no tightness. The sync pin widens with it, and stops scoping itself by issue. That scoping rested on a premise #272 falsified -- the kana members were classification-only, keyed by no default -- and comparing against the whole _SCRIPT_RANGES table is the stronger promise anyway: a new script now fails this test until someone decides in writing whether the rule should cover it, which the added set assertion says out loud. Both corpora still contain no CJK name at all, so the rule classifies nothing in a real run; the harness passes 654 names with 18 classified diffs and none unexplained. That blind spot is #295's subject, cited in the rule's comment, and it is why the behavioral guarantee lives in the case table rather than here. corpus_issues.jsonl is deliberately not regenerated: a dry run adds three Latin strings from #296 and, as #295 predicts, no CJK -- unrelated tracker content that belongs in its own reviewable act. Co-Authored-By: Claude Fable 5 --- tests/v2/test_regex_sync.py | 51 ++++++++++++++---------- tools/differential/expected_changes.toml | 31 ++++++++++---- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index f203088..476b25d 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -164,49 +164,56 @@ def test_comma_char_matches_the_pipeline_comma_set() -> None: def test_differential_cjk_rule_matches_the_script_ranges() -> None: - """The #271 rule in tools/differential/expected_changes.toml hand- - copies the CJK spans from _vocab._SCRIPT_RANGES into a character + """The CJK rule in tools/differential/expected_changes.toml hand- + copies the script spans from _vocab._SCRIPT_RANGES into a character class. A TOML file cannot import the constant, so this is the one copy with no possible alternative -- and the one whose divergence is quietest, because the harness is run by hand rather than in CI. Both failure directions matter, which is why this compares sets rather than checking coverage. A span MISSING from the class turns - an intended #271 change into an UNEXPLAINED diff (a release + an intended #271/#272 change into an UNEXPLAINED diff (a release blocker for the wrong reason); a span that should not be there silently classifies a real regression as intended, which is the failure the whole harness exists to prevent. - Han's astral block is out of scope on both sides. The rule omits - it deliberately -- no corpus name reaches it, see the comment - there -- so the comparison runs over the BMP spans only, and a new - BMP script added to _SCRIPT_RANGES for the SAME #271 behavior - (family-first order / hangul segmentation) still fails here until - the rule covers it. - - #272 added HIRAGANA/KATAKANA to _SCRIPT_RANGES for an unrelated - reason (kana classification for effective_script's license, not - the order-flip or segmentation #271's diff rule explains -- no - default keys on either kana member), so the comparison is scoped - to the #271 scripts by name rather than every table entry; a - future script added for #271's own reason still must extend both - sides, same as before. + Every table entry is in scope. The rule covered HAN and HANGUL + alone while the kana members existed only for classification, but + #272 gave HIRAGANA a default order entry and made the kana blocks + part of the same first/middle/last diff shape the rule explains, + so scoping it by issue no longer draws a real line. Comparing + against the whole table is also the stronger promise: a script + added to _SCRIPT_RANGES for ANY reason fails here until someone + decides, in writing, whether the rule should cover it. + + Han's astral block is the single exception, out of scope on both + sides. The rule omits it deliberately -- no corpus name reaches + it, see the comment there -- so the comparison runs over the BMP + spans only. """ toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_changes.toml") rules = tomllib.loads(toml_path.read_text())["change"] - matched = [r for r in rules if "#271" in r["issue"]] + matched = [r for r in rules + if "#271" in r["issue"] or "#272" in r["issue"]] assert len(matched) == 1, ( - f"expected exactly one #271 rule in {toml_path.name}, " + f"expected exactly one CJK rule in {toml_path.name}, " f"found {len(matched)}") + # A new table entry must force an explicit decision rather than + # quietly widening (or failing to widen) the rule above. + assert set(_vocab._SCRIPT_RANGES) == { + Script.HAN, Script.HANGUL, Script.HIRAGANA, Script.KATAKANA}, ( + "a Script joined _SCRIPT_RANGES: decide whether the " + f"differential rule in {toml_path.name} should cover it, then " + "update this assertion") declared = { (int(lo, 16), int(hi, 16)) for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", matched[0]["name_regex"])} expected = {span - for script in (Script.HAN, Script.HANGUL) - for span in _vocab._SCRIPT_RANGES[script] + for spans in _vocab._SCRIPT_RANGES.values() + for span in spans if span[1] <= 0xFFFF} assert declared == expected, ( - f"{toml_path.name}'s #271 name_regex declares {sorted(declared)}; " + f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index 050ebf7..bfe0da0 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -7,25 +7,40 @@ # ahead of `fields`-only ones before matching. [[change]] -issue = "fix(#271) native-script CJK: family-first order + hangul segmentation" +issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentation, the kana license and the nakaguro" # '毛 泽东', '김민준': script_orders flips first/last for a name written # wholly in Han or Hangul, and the Korean surnames that now ship as # default vocabulary additionally split an unspaced hangul token into -# last + first. Seeded from the fix(#271) rows in tests/v2/cases.py. +# last + first. '高橋 みなみ', 'マイケル・ジャクソン': #272 extends the +# same order table to kana-bearing Japanese names and makes the +# katakana middle dot a token separator, which moves pieces between the +# same three fields. Seeded from the fix(#271) and fix(#272) rows in +# tests/v2/cases.py. +# +# One rule, not two, because it is one diff shape: both issues move +# name pieces between first/middle/last on native-script CJK input, and +# splitting the class by issue would need a rule per script with no +# gain in tightness (the fields list is already the narrow half). # # Scoped by script, which is exactly the scope of the behavior -- -# neither change can touch a name with no CJK character in it. Han's +# nothing here can touch a name with no CJK character in it. Han's # astral block (U+20000-U+323AF) is deliberately left out of the class: # no name in either corpus reaches it, and a rule should be no wider # than the diffs it has to explain. Extend it -- and the sync pin in # tests/v2/test_regex_sync.py, which fails if the two disagree -- if -# one ever does. +# one ever does. The kana spans are BMP-only for a different reason: +# supplementary-plane kana is hentaigana and other archaic forms no +# modern name uses, so _SCRIPT_RANGES does not list it either. # # Expected to match nothing against the current corpora, which contain -# no CJK at all: build_issues_corpus.py requires an internal space, and -# unspaced names are the shape this classifies. Kept for the reason the -# suffix-delimiter rule below is kept -- ready the moment a name lands. -name_regex = "[\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3]" +# no CJK at all -- build_issues_corpus.py requires an internal space, +# and unspaced names are the shape this classifies. That blind spot is +# tracked as #272's and #271's shared gap in issue #295, which also +# records the provenance decision a fix needs; until it is closed the +# behavioral guarantee lives in the fix(#271)/fix(#272) rows of +# tests/v2/cases.py, and this rule is kept ready for the moment a CJK +# name lands in a corpus. +name_regex = "[\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3]" fields = ["first", "middle", "last"] [[change]] From 3b5307630073aafc67bf537e891c6dcb9d9a6a79 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 11:06:16 -0700 Subject: [PATCH 11/14] Classify the ideographic iteration mark as Han (#272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 々 (U+3005) repeats the preceding kanji and carries Script=Common under UAX #24, so it fell outside every range in _SCRIPT_RANGES. That made 佐々木 a mixed-script token: "佐々木 太郎" REVERSED, giving given 佐々木 and family 太郎, and the token never gated into segmentation. 佐々木 (Sasaki) is a top-20 Japanese surname, and 野々村 and 奈々 are ordinary names, so this was not an exotic corner. The mark joins the HAN ranges on the same block-style reasoning U+30FB already gets -- it can only appear inside a Han-written name, being a repeat of the character before it -- along with all three hand-copies the sync pins police. Verified with the extra rather than assumed: namedivider reads 佐々木健 as 佐々木 + 健, so the integration assertion pins the real answer. 佐 is absent from the zh surname list, so the zh pack's reading of these names does not change. Also from the branch review: the differential rule gains U+FF65, whose halfwidth transcriptions do split ('ﾏｲｹﾙ･ｼﾞｬｸｿﾝ') even though halfwidth kana is deliberately unclassified -- the sync pin names that one sanctioned extra rather than relaxing to a subset check, so a second source of divergence still fails. The rest of the halfwidth block stays out: a dotless 'ﾏｲｹﾙ ｼﾞｬｸｿﾝ' is byte-identical on both sides, measured, and covering it would pre-excuse a future regression. Plus four documentation corrections: the nakaguro's exemption from both policy opt-outs stated in AGENTS.md where the doctrine lives, usage.rst's forward pointer widened to kana, _tokenize.py's dangling promise of a follow-up issue replaced with the standing reason, and the available() tuple moved off the #271 bullet that predates ja. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 2 +- docs/release_log.rst | 4 ++-- docs/usage.rst | 6 +++--- nameparser/_pipeline/_tokenize.py | 10 ++++++--- nameparser/_pipeline/_vocab.py | 27 +++++++++++++++--------- nameparser/locales/ja.py | 2 +- nameparser/locales/zh.py | 4 ++-- tests/v2/cases.py | 8 +++++++ tests/v2/test_locales.py | 7 ++++++ tests/v2/test_parser.py | 14 ++++++++++++ tests/v2/test_regex_sync.py | 19 ++++++++++++++++- tools/differential/expected_changes.toml | 14 +++++++++++- 12 files changed, 93 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8ca3c5a..3c25651 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,7 @@ logging.getLogger('HumanName').setLevel(logging.DEBUG) The library has two layers: `nameparser/config/` (data) and `nameparser/parser.py` (logic). -**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Four defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. +**Design philosophy — positional and language-agnostic.** The parser assigns parts by *position* plus small sets of words that join to neighbors; it never detects language. A name's language can't be reliably inferred from Latin-script transliteration ("Ali" is Arabic or Italian; "Van"/"Della"/"Bin" are first names in some cultures, particles in others), so language-specific rules belong in opt-in `Constants` config, never global defaults. Many "wrong for language X" reports (#133, #150, #130, #85, #103, #146, #83) are irreducible ambiguities — e.g. `de Mesnil` (want last name) vs `Van Johnson` (want first name) are the same `[prefix][word]` shape. Before adding a rule, confirm it doesn't break the opposite case (run the full suite — Portuguese and "Van Johnson" tests are the usual canaries). **The one scoped exception (2.1, #271/#272): script-conditional behavior is permitted exactly where the SCRIPT ITSELF — not statistics about it — determines the convention.** The never-detect-language rule above is about Latin *transliteration*, where the signal genuinely is destroyed; native script is a different question, and it is answered per behavior rather than per script. Four defaults fall out of it. Wholly-Han, wholly-Hangul and kana-licensed names read family-first (`Policy.script_orders`) — no language detection needed, because zh and ja both write family-first in native script, so order cannot be misread even though the language is unknowable. Unspaced hangul splits into surname + given name (`config/surnames.py` ships the Korean census list as DEFAULT vocabulary) — nothing but Korean is written in hangul and the surnames are a closed census set, and the vocabulary is self-selecting besides: a hangul entry can only ever match hangul text. Hiragana licenses Japanese (#272) — a name whose characters stay inside Han∪kana while carrying at least one kana cannot be Chinese (the kana rules it out) and is not a transcription (foreign names are transcribed in katakana ALONE, マイケル has no kanji), so 高橋みなみ and 山田 エミ read family-first too; mechanically they resolve to the HIRAGANA entry, the license's carrier key. PURE katakana is excluded and keeps the positional default: マイケル・ジャクソン is a transcribed foreign name in its source order. The nakaguro belongs to the same doctrine but is decided a layer down: U+30FB and its halfwidth twin U+FF65 separate tokens like whitespace, unconditionally and in tokenize, so neither policy opt-out (`script_orders={}`, `segment_scripts=()`) reaches it — the codepoints are CJK-only and appear in no other script's names, which is what licenses a tokenize-level rule where U+00B7 (also the Catalan punt volat, interior to Gal·la) would need a guard. Han segmentation stays OPT-IN (`locales.ZH` for Chinese, `locales.JA` for Japanese) — a zh surname list corrupts Japanese kanji names, since 高 is a common Chinese surname and 高橋一郎 would split 高+橋一郎 where the correct reading is 高橋+一郎; no surname list divides a kanji name at all, so `locales.JA` activates the stage and a pluggable `Parser(segmenter=...)` does the dividing. Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely order-ambiguous and stays governed by `name_order` and opt-in packs. Before adding a script-conditional rule, work out which of the three it is — certain, certain for this one behavior only, or a statistical guess wearing a script's clothes. ### Configuration layer (`nameparser/config/`) diff --git a/docs/release_log.rst b/docs/release_log.rst index 0a82fd0..bc018a1 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -4,10 +4,10 @@ Release Log **East Asian name support** - - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese data goes through ``locales.JA`` and its segmenter instead. It sets no name order, since native-script Han already reads family-first without it. ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (#271) + - Add the Chinese locale pack ``locales.ZH`` — opt-in Han segmentation for unspaced names like ``毛泽东``, with the surname vocabulary it needs. A pack rather than a default because a Chinese surname list corrupts the Japanese names written in the same characters (``高橋一郎`` would split ``高`` + ``橋一郎``); Japanese data goes through ``locales.JA`` and its segmenter instead. It sets no name order, since native-script Han already reads family-first without it (#271) - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) - - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use (closes #272) + - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use. With both packs registered, ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (closes #272) - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest (#272) - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) diff --git a/docs/usage.rst b/docs/usage.rst index a133f00..f58631a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -52,9 +52,9 @@ given-then-family: 'Doe' For family-first input *without* a comma — common outside Europe — set -``name_order``; see :doc:`customize`. Names written in Han or Hangul -are the exception that needs no setting at all: see `East Asian -names`_ below. +``name_order``; see :doc:`customize`. Names written in Han or Hangul, +and Japanese names written in kanji and kana, are the exception that +needs no setting at all: see `East Asian names`_ below. Words that attach to their neighbors -------------------------------------- diff --git a/nameparser/_pipeline/_tokenize.py b/nameparser/_pipeline/_tokenize.py index 28460e4..0d4643d 100644 --- a/nameparser/_pipeline/_tokenize.py +++ b/nameparser/_pipeline/_tokenize.py @@ -44,9 +44,13 @@ # The katakana middle dot and its halfwidth twin divide the parts of # a foreign name transcribed into katakana (マイケル・ジャクソン) -- # native names never contain them, so they separate unconditionally, -# like whitespace (amendment 2026-07-29 section 1b). U+00B7 is -# deliberately NOT here: it is also the Catalan punt volat, interior -# to real names (Gal·la) -- a follow-up issue owns that decision. +# like whitespace (amendment 2026-07-29 section 1b). The Chinese +# interpunct U+00B7 (威廉·莎士比亚) is deliberately NOT here: it is +# also the Catalan punt volat, which sits INSIDE a single name piece +# (Gal·la), so separating on it unconditionally would break names it +# has no business touching. Admitting it would need a flanked-by-CJK +# guard, which is a different rule from this set's "these codepoints +# are always separators". _NAME_DOT_SEPARATORS = frozenset({"\u30FB", "\uFF65"}) diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 7054bde..26bc8fa 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -37,14 +37,21 @@ # per-char on the _EMOJI_RANGES precedent in _tokenize.py, on the # theory that a range test needs no regex; measured at token scale the # compiled regex wins by 3-9x, and by 89x on long tokens.) -# HAN: the URO plus Extension A, the compatibility block, -# and the supplementary-plane block (Ext B-I + CJK Compat Ideographs -# Supplement, 0x20000-0x323AF) -- rare surnames are the biggest real -# source of supplementary-plane hanzi in personal names (e.g. 𠮷田's -# 𠮷, U+20BB7), so leaving them out silently mis-orders those names; -# unassigned gaps inside the span are harmless, since no real name -# contains an unassigned codepoint. HANGUL: precomposed syllables -# only -- modern Korean text never writes names as bare jamo. +# HAN: the ideographic iteration mark U+3005, the URO plus Extension +# A, the compatibility block, and the supplementary-plane block +# (Ext B-I + CJK Compat Ideographs Supplement, 0x20000-0x323AF) -- +# rare surnames are the biggest real source of supplementary-plane +# hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them +# out silently mis-orders those names; unassigned gaps inside the span +# are harmless, since no real name contains an unassigned codepoint. +# U+3005 々 is the block-vs-Script case again (it is Script=Common +# under UAX #24, like U+30FB below): it REPEATS the preceding kanji +# and appears only inside Han-written names -- 佐々木 (Sasaki, a +# top-20 Japanese surname), 野々村, 奈々 -- so a token carrying it is +# as Han as the character it iterates. Omitting it made 佐々木 a +# mixed-script token: the name reversed and never gated into +# segmentation. HANGUL: precomposed syllables only -- modern Korean +# text never writes names as bare jamo. # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There # IS a supplementary-plane kana repertoire (Kana Supplement, Kana # Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, 311 assigned @@ -75,8 +82,8 @@ # (dict iteration order), so an overlapping future script would make # the result order-dependent instead of well-defined. _SCRIPT_RANGES: dict[Script, tuple[tuple[int, int], ...]] = { - Script.HAN: ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), - (0x20000, 0x323AF)), + Script.HAN: ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), (0x20000, 0x323AF)), Script.HANGUL: ((0xAC00, 0xD7A3),), Script.HIRAGANA: ((0x3040, 0x309F),), Script.KATAKANA: ((0x30A0, 0x30FF),), diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index e0e215f..8ee21da 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -81,7 +81,7 @@ # kana-licensed composite the pack acts on (山田エミ) -- DEVIATES must # declare it, and the adapter must not decline it. A pure-katakana # token never reaches either: the stage's own gate stops it. -_JA_RANGES = ((0x3040, 0x309F), (0x30A0, 0x30FF), +_JA_RANGES = ((0x3005, 0x3005), (0x3040, 0x309F), (0x30A0, 0x30FF), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)) diff --git a/nameparser/locales/zh.py b/nameparser/locales/zh.py index 30d8e06..1ac8cbe 100644 --- a/nameparser/locales/zh.py +++ b/nameparser/locales/zh.py @@ -101,8 +101,8 @@ # _pipeline/_vocab.py's _SCRIPT_RANGES[Script.HAN] (layering forbids # a pack importing the pipeline; the sync test in # tests/v2/test_locales.py pins the equality). -_HAN_RANGES = ((0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), - (0x20000, 0x323AF)) +_HAN_RANGES = ((0x3005, 0x3005), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), (0x20000, 0x323AF)) def DEVIATES(name: str) -> bool: diff --git a/tests/v2/cases.py b/tests/v2/cases.py index f0a48bf..c8b6e44 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -613,6 +613,14 @@ def __post_init__(self) -> None: notes="parity: katakana deliberately has no entry, so the " "positional default holds -- transcribed foreign names " "keep source order"), + Case("ja_iteration_mark_is_han", "佐々木 太郎", + {"family": "佐々木", "given": "太郎"}, + classification="fix(#272)", + notes="々 (U+3005, the ideographic iteration mark) repeats " + "the preceding kanji and is Script=Common under UAX " + "#24; it counts as Han here, or 佐々木 -- a top-20 " + "Japanese surname -- would be a mixed-script token and " + "reverse"), Case("ja_nakaguro_inside_a_nickname", "山田 太郎 (マイケル・ジャクソン)", {"family": "山田", "given": "太郎", diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 0ac5213..a339b35 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -246,6 +246,12 @@ def test_ja_end_to_end() -> None: # kana-licensed composite: gated in through the HIRAGANA entry n = p.parse("高橋みなみ") assert (n.family, n.given) == ("高橋", "みなみ") + # the iteration mark 々 gates in as Han and namedivider reads the + # repeated character correctly -- verified against 0.4.1 rather + # than assumed, since 々 is Script=Common and a divider could + # plausibly have choked on it or cut beside it + n = p.parse("佐々木健") + assert (n.family, n.given) == ("佐々木", "健") @_needs_ja @@ -576,6 +582,7 @@ def _default_corpus() -> list[str]: "高橋みなみ", # kanji + hiragana: the kana license's carrier case "山田エミ", # kanji + katakana: licensed as a MIXED token, where # pure katakana (マイケル) is deliberately not + "佐々木健", # the iteration mark 々 inside a top-20 surname ] diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index edbcafc..6dc259f 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -483,6 +483,20 @@ def test_kana_licensed_names_read_family_first_by_default() -> None: assert parse("高橋みなみ").family == "高橋みなみ" +def test_iteration_mark_counts_as_han() -> None: + # 々 (U+3005) repeats the preceding kanji and is Script=Common + # under UAX #24, so classifying by the Script property would put a + # 佐々木 token in no script at all: the name reversed, and the + # token never reached the segmentation gate. It is in the HAN + # ranges by BLOCK-style reasoning, the same call U+30FB gets. + n = parse("佐々木 太郎") + assert (n.family, n.given) == ("佐々木", "太郎") + n = parse("野々村 真") + assert (n.family, n.given) == ("野々村", "真") + # and a lone one takes the family role like any other Han token + assert parse("奈々").family == "奈々" + + def test_pure_katakana_stays_positional() -> None: # transcribed foreign names keep their original (usually Western) # order; katakana alone licenses nothing diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 476b25d..6a9c72f 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -190,6 +190,17 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: sides. The rule omits it deliberately -- no corpus name reaches it, see the comment there -- so the comparison runs over the BMP spans only. + + The rule is also WIDER than the table by exactly one span, which + the equality has to know about or it would just fail forever. The + halfwidth middle dot U+FF65 changes parses without being + classified as anything: tokenize separates on it, so a halfwidth + transcription splits where 1.4 kept one token, while halfwidth + kana stays out of _SCRIPT_RANGES on purpose. Naming that span here + rather than relaxing the comparison to a subset check is what + keeps the pin honest in both directions: a THIRD source of + divergence still fails, and the one sanctioned difference has to + be written down to exist. """ toml_path = (Path(__file__).parents[2] / "tools" / "differential" / "expected_changes.toml") @@ -210,10 +221,16 @@ def test_differential_cjk_rule_matches_the_script_ranges() -> None: (int(lo, 16), int(hi, 16)) for lo, hi in re.findall(r"\\u([0-9A-Fa-f]{4})-\\u([0-9A-Fa-f]{4})", matched[0]["name_regex"])} + # the one span the rule carries that no Script claims (see above) + halfwidth_dot = (0xFF65, 0xFF65) + assert not any(lo <= halfwidth_dot[0] <= hi + for spans in _vocab._SCRIPT_RANGES.values() + for lo, hi in spans), ( + "U+FF65 is classified now; drop it from the sanctioned extras") expected = {span for spans in _vocab._SCRIPT_RANGES.values() for span in spans - if span[1] <= 0xFFFF} + if span[1] <= 0xFFFF} | {halfwidth_dot} assert declared == expected, ( f"{toml_path.name}'s CJK name_regex declares {sorted(declared)}; " f"_SCRIPT_RANGES' BMP spans are {sorted(expected)}") diff --git a/tools/differential/expected_changes.toml b/tools/differential/expected_changes.toml index bfe0da0..0711762 100644 --- a/tools/differential/expected_changes.toml +++ b/tools/differential/expected_changes.toml @@ -32,6 +32,18 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # supplementary-plane kana is hentaigana and other archaic forms no # modern name uses, so _SCRIPT_RANGES does not list it either. # +# U+FF65 is the one span here that _SCRIPT_RANGES does NOT have, and +# it is deliberate: the halfwidth middle dot separates tokens like its +# fullwidth twin, so 'ﾏｲｹﾙ･ｼﾞｬｸｿﾝ' splits where 1.4 left one token, +# but halfwidth kana is excluded from CLASSIFICATION on purpose (a +# separate normalization problem). The rest of the halfwidth block +# (U+FF66-U+FF9F) is left out for the usual tightness reason, and it +# was measured rather than assumed: a dotless halfwidth name such as +# 'ﾏｲｹﾙ ｼﾞｬｸｿﾝ' is byte-identical on both sides, so covering the +# block would pre-excuse a future regression on a shape that is +# parity today. The dot is the whole mechanism; the dot is the whole +# span. +# # Expected to match nothing against the current corpora, which contain # no CJK at all -- build_issues_corpus.py requires an internal space, # and unspaced names are the shape this classifies. That blind spot is @@ -40,7 +52,7 @@ issue = "fix(#271/#272) native-script CJK: family-first order, hangul segmentati # behavioral guarantee lives in the fix(#271)/fix(#272) rows of # tests/v2/cases.py, and this rule is kept ready for the moment a CJK # name lands in a corpus. -name_regex = "[\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3]" +name_regex = "[\\u3005-\\u3005\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65]" fields = ["first", "middle", "last"] [[change]] From 6b0eae02a995ed552c3f66d971c106c8e4c1ed48 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 11:47:39 -0700 Subject: [PATCH 12/14] Pin the ja adapter against a stub so coverage sees it (#272) The adapter's guard stack ran only in the extra-gated integration tests, which the coverage-bearing CI job never executes -- so codecov/patch read locales/ja.py at 55% while ja-extra proved it green. A stubbed namedivider module exercises every branch in the plain suite, including the two defensive paths the real 0.4.x library cannot reach, plus the gbdt selection. Also pins the two remaining uncovered new lines: parser_for's base type check and Segmentation's confidence TYPE error. Co-Authored-By: Claude Fable 5 --- tests/v2/test_locales.py | 75 ++++++++++++++++++++++++++++++++++++++++ tests/v2/test_parser.py | 5 +++ tests/v2/test_types.py | 3 ++ 3 files changed, 83 insertions(+) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index a339b35..acc6f71 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -3,6 +3,8 @@ import functools import importlib.util import re +import sys +import types from collections.abc import Callable, Iterable from types import ModuleType @@ -215,6 +217,79 @@ def test_ja_segmenter_without_extra_raises_helpfully() -> None: locales.ja_segmenter() +class _FakeDivided: + """The shape of namedivider's DividedName, minus namedivider — + lets the adapter's guard stack run in the plain (no-extra) suite, + including the two defensive branches the real 0.4.x library can + never reach (reconstruction failure, an empty side).""" + + def __init__(self, family: str, given: str, score: float) -> None: + self.family, self.given, self.score = family, given, score + + +def _fake_namedivider(monkeypatch: pytest.MonkeyPatch, + divide: "Callable[[str], _FakeDivided]") -> list[str]: + """Install a stub namedivider module and return the gbdt-call log + (empty unless GBDTNameDivider was constructed).""" + constructed: list[str] = [] + + class _Basic: + def __init__(self) -> None: + constructed.append("basic") + + def divide_name(self, text: str) -> _FakeDivided: + return divide(text) + + class _GBDT(_Basic): + def __init__(self) -> None: + constructed.append("gbdt") + + stub = types.ModuleType("namedivider") + stub.BasicNameDivider = _Basic # type: ignore[attr-defined] + stub.GBDTNameDivider = _GBDT # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "namedivider", stub) + return constructed + + +def test_ja_adapter_guard_stack_against_a_stub( + monkeypatch: pytest.MonkeyPatch) -> None: + # The adapter's whole contract without the extra: this is what the + # coverage-bearing CI job sees, so the guards are pinned here and + # not only behind the nameparser[ja] integration tests. + divide = _fake_namedivider( + monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], 0.5)) + seg = locales.ja_segmenter() + assert divide == ["basic"] + answer = seg("山田太郎") + assert answer is not None + assert answer.splits == (2,) and answer.confidence == 0.5 + assert seg("김민준") is None # outside the repertoire + assert seg("林") is None # namedivider raises below 2 + + +def test_ja_adapter_defensive_branches_against_a_stub( + monkeypatch: pytest.MonkeyPatch) -> None: + # reconstruction failure -> decline; an empty side -> the stated + # "confidently undivided" opinion; an out-of-range score -> clamped + # so Segmentation's [0, 1] validation cannot raise mid-parse + _fake_namedivider( + monkeypatch, lambda text: _FakeDivided("外", "れ", 0.5)) + assert locales.ja_segmenter()("山田太郎") is None + _fake_namedivider( + monkeypatch, lambda text: _FakeDivided(text, "", 1.0000001)) + whole = locales.ja_segmenter()("山田太郎") + assert whole is not None + assert whole.splits == () and whole.confidence == 1.0 + + +def test_ja_adapter_gbdt_flag_selects_the_gbdt_divider( + monkeypatch: pytest.MonkeyPatch) -> None: + divide = _fake_namedivider( + monkeypatch, lambda text: _FakeDivided(text[:1], text[1:], 1.0)) + locales.ja_segmenter(gbdt=True) + assert divide == ["gbdt"] + + def test_ja_ranges_stay_in_sync_with_vocab() -> None: # the twin of the zh pin above: ja.py hand-copies the three # Japanese-repertoire scripts' spans for DEVIATES and for the diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 6dc259f..a6902d0 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -634,6 +634,11 @@ def seg(text: str) -> Segmentation | None: assert p.segmenter is seg +def test_parser_for_rejects_a_non_parser_base() -> None: + with pytest.raises(TypeError, match="base must be a Parser"): + parser_for(locales.get("zh"), base=5) # type: ignore[arg-type] + + def test_parser_for_takes_a_segmenter_keyword() -> None: def seg(text: str) -> Segmentation | None: return None diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 12a17c7..aad8ad9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -87,6 +87,9 @@ def test_segmentation_validates_splits() -> None: Segmentation(("2",)) # type: ignore[arg-type] with pytest.raises(ValueError, match="confidence"): Segmentation((2,), confidence=1.5) + # wrong TYPE is the TypeError branch, distinct from the range check + with pytest.raises(TypeError, match="confidence"): + Segmentation((2,), confidence="high") # type: ignore[arg-type] def test_segmentation_rejects_mappings_and_non_iterables() -> None: From 94fd02e4fc4745b7f004f59c6e8390f445721ea0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 12:20:37 -0700 Subject: [PATCH 13/14] Apply the five-agent PR review round (#272) Two behavior changes, drawing one line the segmenter path did not previously state: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its third-party library declines. A segmenter answer cutting at or past the end of the token it was handed now raises ValueError naming the offset and the length, matching the wrong-answer-type TypeError beside it. Both are stage-detectable bugs in user code, inside the already-declared totality exception; the old silent decline made an off-by-one segmenter undebuggable, since every answer it gave simply vanished and the name merely looked undivided. The ja adapter's confidence clamp becomes epsilon-tolerant-else- decline. Float noise at the edge of namedivider's softmax still clamps into [0, 1] -- sized for float32, whose eps is ~1.2e-7, since rule-based answers score exactly 1.0 and a tighter band would reject a correct division that came back one ulp high. A score outside that band does not clamp: a divider reporting 87.0 is broken and its division is worth no more than its score, so the adapter declines, which keeps the parse safe and observable by the report's absence where the old clamp mapped garbage to "certain" and silenced it. NaN declines through the same comparison. Also: parser_for's segmenter takes the UNSET sentinel, so an explicit None clears the base's rather than reading as "not given"; the SEGMENTATION docstring covers both of its sources; a bare-string guard on Segmentation.splits; the U+3005 rationale corrected in four files (it is Script=Han, and the BLOCK table is what needed the singleton); the composition claim qualified wherever it appears; and tests for the lot, including the range-disjointness pin, the capture-the-argument and unactivated-neighbour stage cases, and the lone-hiragana, astral-kanji and ZH+JA-collision integrations. Co-Authored-By: Claude Fable 5 --- .github/workflows/python-package.yml | 10 ++- AGENTS.md | 2 +- docs/release_log.rst | 2 +- docs/usage.rst | 18 +++-- nameparser/_parser.py | 31 +++++--- nameparser/_pipeline/_script_segment.py | 65 ++++++++++++---- nameparser/_pipeline/_vocab.py | 43 ++++++----- nameparser/_types.py | 39 +++++++--- nameparser/locales/ja.py | 63 +++++++++++++--- tests/v2/cases.py | 6 +- tests/v2/pipeline/test_script_segment.py | 68 +++++++++++++---- tests/v2/pipeline/test_vocab.py | 22 ++++++ tests/v2/test_cli.py | 13 ++++ tests/v2/test_locales.py | 94 +++++++++++++++++++++++- tests/v2/test_parser.py | 29 ++++++-- tests/v2/test_types.py | 23 ++++++ 16 files changed, 430 insertions(+), 98 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1ca6e0c..943f2ee 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -89,4 +89,12 @@ jobs: - name: Install dependencies run: uv sync --group dev --extra ja - name: Run Tests - run: uv run pytest tests/v2/ -q + # The import check first: every test the extra adds is guarded by + # a find_spec("namedivider") skipif, so a job where the extra + # silently failed to install is a job where all of them SKIP and + # the step still passes green. This line is what makes that + # failure loud -- without it the job proves nothing on the day it + # matters. + run: | + uv run python -c "import namedivider" + uv run pytest tests/v2/ -q diff --git a/AGENTS.md b/AGENTS.md index 3c25651..d34665d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. -- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is the optional `Parser(segmenter=...)` hook — a Parser holding a callable pickles iff that callable does, and the same hook is parse-totality's one exception: a user-supplied segmenter's own exceptions propagate rather than being swallowed as content errors). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is the optional `Parser(segmenter=...)` hook — a Parser holding a callable pickles iff that callable does, and the same hook is parse-totality's one exception. Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, and `test_regex_sync.py`, which pins every hand-copied pattern or codepoint table against its source wherever the copy lives — including copies outside the package), names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/docs/release_log.rst b/docs/release_log.rst index bc018a1..0cae824 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - Add ``Lexicon.surnames``, ``Policy.script_orders``, ``Policy.segment_scripts``, the ``Script`` enum and the ``DEFAULT_SCRIPT_ORDERS`` constant to the public API. This is the first behavior nameparser keys on the script a name is written in; it is allowed only where the script itself settles a convention, never as a proxy for guessing the language (#271) - Add ``AmbiguityKind.SEGMENTATION``, reported when a surname split had a vocabulary-supported alternative: ``"남궁민수"`` is 남궁 + 민수 by the compound surname but 남 + 궁민수 by the single-syllable one, and longest-match had to pick. A name with only one possible split decided nothing and reports nothing (#271) - Add the Japanese locale pack ``locales.JA`` and the segmenter factory ``locales.ja_segmenter()``, which together divide an unspaced Japanese name: ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` reads ``山田太郎`` as family ``山田``, given ``太郎``. The two halves are separate because no surname list can do this job — family and given names draw on the same kanji and the reading, not the spelling, decides most divisions — so the pack activates the stage and a third-party divider performs it. ``ja_segmenter()`` wraps `namedivider-python `_, installed with the new ``nameparser[ja]`` extra; the core stays dependency-free, and ``ja_segmenter(gbdt=True)`` selects namedivider's more accurate gradient-boosted model, which downloads its data on first use. With both packs registered, ``locales.available()`` is now ``('ja', 'ru', 'tr_az', 'zh')`` (closes #272) - - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest (#272) + - Add ``Segmentation`` and the ``Segmenter`` type alias to the public API, plus the keyword-only ``Parser(segmenter=...)`` hook they describe: any callable from a token's text to a ``Segmentation`` (the interior offsets to cut at, and a confidence) or ``None`` to decline. It is consulted only for scripts listed in ``Policy.segment_scripts``, and only where the surname vocabulary declined first, so ``parser_for(locales.ZH, locales.JA, segmenter=...)`` composes — a listed Chinese surname wins, the segmenter takes the rest. Read that composition with the zh pack's own warning still attached: vocabulary-first means a Japanese kanji name opening on a listed Chinese surname never reaches the segmenter, so ``高橋一郎`` still splits ``高`` + ``橋一郎`` under the stack exactly as it does under ``locales.ZH`` alone. The two packs are alternatives, one per corpus; stack them only for genuinely mixed data that accepts that trade (#272) - Add the ``Script`` members ``HIRAGANA`` and ``KATAKANA``. Two members rather than one ``KANA`` because the parser treats them differently: hiragana never transcribes a foreign name, while a wholly-katakana name usually is one (#272) **Breaking Changes** diff --git a/docs/usage.rst b/docs/usage.rst index f58631a..61f249a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -296,11 +296,19 @@ the presumption Japanese practice makes; that is an accepted presumption, not a measurement, so a two-character token is the shape to check first if a division looks wrong. -A segmenter that answers outside the token it was given — a cut past -the end of the text, or pieces that do not reconstruct the input — is -declined silently and the token is left whole. Exceptions are the one -thing that does *not* stay inside the parse: a segmenter is your code, -so its errors propagate rather than being absorbed as content errors. +A segmenter that answers outside the token it was given — a cut at or +past the end of the text — has violated the protocol, and the parse +says so rather than hiding it: it raises ``ValueError`` naming the +offending offset and the token's length, the way an answer of the +wrong type raises ``TypeError``. Declining silently would leave an +off-by-one segmenter invisible; every answer it gave would vanish and +the name would merely look undivided. The shipped ``ja_segmenter()`` +does decline — returns ``None``, token left whole — for the cases that +are *not* protocol violations: text outside the Japanese repertoire, +text too short to divide, an answer that fails to reconstruct its +input, and a score outside [0, 1]. Exceptions are the one thing that +does *not* stay inside the parse: a segmenter is your code, so its +errors propagate rather than being absorbed as content errors. The command line takes the pack but not the segmenter: ``python -m nameparser --locale ja`` has no way to attach one, so it activates diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 7625855..d7d73a7 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -18,7 +18,7 @@ from nameparser._pipeline import run from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState -from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch +from nameparser._policy import UNSET, Policy, PolicyPatch, _Unset, apply_patch from nameparser._types import ( FOLDED_TAG, ParsedName, Segmenter, Token, _guarded_getstate, _guarded_setstate, _validated_field_strings, @@ -187,7 +187,7 @@ def parse(text: str) -> ParsedName: def parser_for(*locales: Locale, base: Parser | None = None, - segmenter: Segmenter | None = None) -> Parser: + segmenter: Segmenter | None | _Unset = UNSET) -> Parser: """Lexicon fragments unioned left-to-right onto base's; policy patches applied left-to-right (later wins; set-valued fields union per the patch metadata). Validation errors raised while applying a @@ -199,9 +199,14 @@ def parser_for(*locales: Locale, base: Parser | None = None, A ``segmenter`` is passed straight through to the built Parser -- ``parser_for(locales.JA, segmenter=locales.ja_segmenter())`` is how a pack and a segmenter combine, since packs are pure data and - cannot supply one. Given explicitly it OVERRIDES base's (later - wins, the rule scalar policy fields follow); omitted, base's - carries through unchanged.""" + cannot supply one. The argument has THREE states, the same + :data:`~nameparser.UNSET` spelling a PolicyPatch field uses, because + None is a meaningful value here and not an absence: omitted (UNSET) + carries base's segmenter through unchanged; a callable OVERRIDES + base's (later wins, the rule scalar policy fields follow); and an + explicit ``None`` CLEARS base's, which is how you derive an + unsegmented parser from a segmented one without rebuilding its + lexicon and policy by hand.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") for loc in locales: @@ -209,13 +214,15 @@ def parser_for(*locales: Locale, base: Parser | None = None, raise TypeError(f"parser_for() takes Locale packs, got {loc!r}") lexicon = base.lexicon if base is not None else Lexicon.default() policy = base.policy if base is not None else Policy() - # Unpacked here with its siblings because the return builds a FRESH - # Parser: any field not listed there silently takes its default, and - # a dropped segmenter would be invisible. None reads as "not given" - # rather than "clear it" (the escape hatch is a plain Parser: - # Parser(lexicon=p.lexicon, policy=p.policy)). - if segmenter is None and base is not None: - segmenter = base.segmenter + # Resolved here rather than at the return because the return builds + # a FRESH Parser: any field not listed there silently takes its + # default, and a dropped segmenter would be invisible. UNSET, not + # None, is what "not given" means -- None is the CLEAR request, and + # collapsing the two would make an explicit + # parser_for(..., segmenter=None) silently inherit the very + # segmenter it was asked to drop. + if segmenter is UNSET: + segmenter = base.segmenter if base is not None else None scalar_setters: dict[str, str] = {} for loc in locales: for f in dataclasses.fields(PolicyPatch): diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 454323c..84edeb2 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -20,8 +20,15 @@ Where the VOCABULARY declines -- no prefix matched -- an optional Parser(segmenter=...) gets the token (#272 amendment 2026-07-29). Vocabulary first, segmenter on decline, so parser_for(ZH, JA, -segmenter=...) composes: a listed surname is a dictionary certainty -and wins, and the segmenter takes what is left. Its Segmentation may +segmenter=...) composes MECHANICALLY: a listed surname is a +dictionary certainty and wins, and the segmenter takes what is left. +Composing is not a free lunch, and the docs qualify it where they +show the stack: the zh pack's own mis-split warning survives +unchanged, because a Japanese kanji name opening on a listed Chinese +surname never reaches the segmenter at all -- 高橋一郎 still splits +高 + 橋一郎 under ZH+JA, exactly as it does under ZH alone. The two +packs are corpus ALTERNATIVES, one per corpus; stacking them is for +genuinely mixed data that accepts that trade. Its Segmentation may cut anywhere and any number of times, which is why the split path below is written for n cuts and the vocabulary hit is simply its one-cut case. One precondition guards it that the vocabulary has no @@ -86,7 +93,7 @@ #: measuring namedivider 0.4.1 over 112 names (#272 Task 5), which #: found a distribution the amendment did not anticipate: the scores #: are BIMODAL, not clustered near 1. A rule-based division (the kana -#: boundary in 高橋みなみ, a two-character name) scores exactly 1.0; +#: boundary in 高橋みなみ, or a two-character name) scores exactly 1.0; #: a kanji-statistics division scores a softmax over the candidate cut #: positions, observed in 0.23-0.68 and driven more by name LENGTH #: (median 0.61 at three characters, 0.37 at five -- the softmax is @@ -98,8 +105,18 @@ #: epistemic argument asks for anyway. 0.9 sits in the empty gap #: (0.68 to 1.0), far from both modes, and reads as "confident" for a #: third-party segmenter with a calibrated score too. Consequence, -#: pinned by tests: a kanji-only Japanese name always carries a -#: SEGMENTATION report, a kana-divided one never does. +#: pinned by tests and stated so it can be checked: every +#: STATISTICALLY divided name carries a SEGMENTATION report, and every +#: RULE-divided one -- the kana boundary, a two-character name, +#: namedivider's specific-name rules -- carries none. +#: One case is worth writing down rather than rediscovering: the +#: two-character division is a PRESUMPTION, not a measurement (usage.rst +#: says exactly that), yet namedivider scores it 1.0 and this floor +#: therefore keeps it silent. That is the divider's own stated +#: certainty, accepted here as-is rather than second-guessed by a +#: length rule of ours. If presumption-reporting is ever wanted, the +#: hook is namedivider's DividedName.algorithm, which names WHICH rule +#: answered -- not the score, which cannot tell the two apart. #: Not configurable in 2.x (YAGNI). _SEGMENTER_CONFIDENCE_FLOOR = 0.9 @@ -270,25 +287,41 @@ def script_segment(state: ParseState) -> ParseState: for j in state.segments[0]): return state # No try/except around the call: the module docstring's totality - # exception. The check below is that same doctrine, curated -- a - # duck-typed answer carrying a .splits of its own would otherwise - # wander into the split path and surface as a ValueError naming - # Token, pointing the reader at nameparser's insides instead of at - # their segmenter. Bounded like every message here: the type's - # NAME, never its contents. + # exception. The two checks below are that same doctrine, curated, + # and they are where the line this module draws is easiest to state: + # a PROTOCOL VIOLATION BY THE SEGMENTER AUTHOR RAISES, while an + # ADAPTER'S DEFENSE AGAINST ITS LIBRARY DECLINES. Both checks here + # are the first kind -- a wrong answer TYPE and an answer indexing + # past the token it was handed are stage-detectable bugs in + # user-supplied code, inside the declared totality exception, so + # they get the same treatment the callable's own exceptions get. + # locales/ja.py is the second kind: its repertoire, length, + # reconstruction and score guards all return None, because what + # they defend against is namedivider answering a question nobody + # asked it, which is a fact about the CONTENT, not a broken + # protocol. Bounded like every message here: the type's NAME, never + # its contents. answer = state.segmenter(text) if answer is not None and not isinstance(answer, Segmentation): + # a duck-typed answer carrying a .splits of its own would + # otherwise wander into the split path and surface as a + # ValueError naming Token, pointing the reader at nameparser's + # insides instead of at their segmenter raise TypeError( f"segmenter must return Segmentation or None, got " f"{type(answer).__name__}") if answer is None or not answer.splits: return state # declined, or confidently one token - # splits[-1] is the max -- Segmentation enforced ascending + # splits[-1] is the max -- Segmentation enforced ascending. The + # upper bound is the half Segmentation cannot check, since it never + # sees the text; an offset at or past the end would make an empty + # piece. Declining silently here (as this did before the review) + # made an off-by-one segmenter undebuggable: every answer it gave + # vanished, and the parse merely looked unsegmented. if answer.splits[-1] >= len(text): - # the upper bound is ours to check (Segmentation never sees the - # text, so it cannot); a violating answer is a buggy segmenter, - # treated as a decline -- never a crash - return state + raise ValueError( + f"segmenter returned splits beyond the token: last offset " + f"{answer.splits[-1]}, token length {len(text)}") conf = answer.confidence detail = None if conf is not None and conf < _SEGMENTER_CONFIDENCE_FLOOR: diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 26bc8fa..3c51c69 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -44,18 +44,23 @@ # hanzi in personal names (e.g. 𠮷田's 𠮷, U+20BB7), so leaving them # out silently mis-orders those names; unassigned gaps inside the span # are harmless, since no real name contains an unassigned codepoint. -# U+3005 々 is the block-vs-Script case again (it is Script=Common -# under UAX #24, like U+30FB below): it REPEATS the preceding kanji -# and appears only inside Han-written names -- 佐々木 (Sasaki, a -# top-20 Japanese surname), 野々村, 奈々 -- so a token carrying it is -# as Han as the character it iterates. Omitting it made 佐々木 a -# mixed-script token: the name reversed and never gated into -# segmentation. HANGUL: precomposed syllables only -- modern Korean +# U+3005 々 is the block-vs-Script case, running the OPPOSITE way to +# U+30FB below: 々 already IS Script=Han under UAX #24 (Scripts.txt +# reads `3005 ; Han`), but it sits in CJK Symbols and Punctuation, +# outside every CJK ideograph block this table spans -- so the +# singleton entry is what a BLOCK table needs to reach a character the +# Script property would have classified correctly for free. It earns +# the reach: 々 repeats the preceding kanji and appears only inside +# Han-written names -- 佐々木 (Sasaki, a top-20 Japanese surname), +# 野々村, 奈々. Omitting it made 佐々木 a mixed-script token: the name +# reversed and never gated into segmentation. +# HANGUL: precomposed syllables only -- modern Korean # text never writes names as bare jamo. # HIRAGANA/KATAKANA (#272): the two kana blocks, each in full. There # IS a supplementary-plane kana repertoire (Kana Supplement, Kana -# Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, 311 assigned -# codepoints) but none of it is WORTH chasing the way Han's astral +# Extended-A/B, Small Kana Extension, U+1AFF0-U+1B16F, a few hundred +# assigned codepoints -- no exact count here, it moves with the +# Unicode version) but none of it is WORTH chasing the way Han's astral # block is: those codepoints are hentaigana and other archaic/ # phonetic-extension forms no modern Japanese name uses, unlike # supplementary Han, which real surnames genuinely need. The Katakana @@ -69,15 +74,17 @@ # This table classifies by Unicode BLOCK, not the UAX #24 Script # property: U+30A0, U+30FB (the middle dot), and U+30FC (the # prolonged sound mark) all carry Script=Common under UAX #24, and the -# combining kana voicing marks U+3099-U+309C are Common/Inherited -- -# yet every one of them is needed here, and block membership, not the -# Script property, is what puts them in range. The katakana block's -# upper end (U+30FF) including the middle dot U+30FB is load-bearing -# for effective_script's kana license below (see its docstring) -- -# tokenize (#272 Task 2b) now turns U+30FB into a token separator -# before this classifier ever sees a split string, but effective_script -# is also called directly on whole strings (e.g. in tests), where the -# dot still classifies as ordinary katakana. The ranges below must stay +# four kana voicing marks U+3099-U+309C split two and two -- U+3099 +# and U+309A are the COMBINING forms (Script=Inherited), U+309B and +# U+309C the spacing ones (Script=Common) -- yet every one of them is +# needed here, and block membership, not the Script property, is what +# puts them in range. The katakana block's upper end (U+30FF) takes in +# the middle dot U+30FB, kept rather than carved out for a smaller +# reason than it looks: tokenize (#272 Task 2b) turns U+30FB into a +# token separator, so no real parse shows this classifier a string +# containing one. It is kept so that a DIRECT whole-string call -- +# effective_script("マイケル・ジャクソン"), which the unit tests make -- +# still classifies instead of returning None. The ranges below must stay # mutually disjoint: single_script returns the FIRST covering entry # (dict iteration order), so an overlapping future script would make # the result order-dependent instead of well-defined. diff --git a/nameparser/_types.py b/nameparser/_types.py index cccf42e..5b9a178 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -240,8 +240,11 @@ class Segmentation: rewrite characters) and an optional confidence in [0, 1]. ``Segmentation(())`` means "confidently one token" -- distinct from returning None, which DECLINES ("I don't know"). The upper bound - (< len(token)) is validated by the consuming stage, which knows the - text.""" + (< len(token)) is the half this class cannot check, never having + seen the text; the consuming stage checks it and RAISES + ``ValueError`` on a violation, the same call it makes on an answer + of the wrong type -- both are protocol bugs in the segmenter, not + facts about the name.""" #: Interior character offsets to split at, ascending. splits: tuple[int, ...] @@ -253,9 +256,19 @@ class Segmentation: __setstate__ = _guarded_setstate def __post_init__(self) -> None: - # The same guard Token.tags carries: a mapping would silently - # contribute only its keys, and a bare int would surface as an - # uncurated "not iterable" from the tuple() below. + # Both guards Token.tags carries, for the same two reasons and + # a third of this field's own: a bare string is iterable, so + # Segmentation("") would sail through as "confidently one + # token" -- an opinion nobody stated -- and Segmentation("23") + # would fail one character deep naming '2' rather than the + # argument. A mapping would silently contribute only its keys, + # and a bare int would surface as an uncurated "not iterable" + # from the tuple() below. + if isinstance(self.splits, str): + raise TypeError( + "Segmentation.splits must be an iterable of integers, " + "not a bare string" + ) if isinstance(self.splits, Mapping): raise TypeError( "Segmentation.splits must be an iterable of integers, " @@ -357,10 +370,18 @@ class AmbiguityKind(StrEnum): #: More comma-separated segments than any recognized name shape; #: the parse is best-effort over the extra segments. COMMA_STRUCTURE = "comma-structure" - #: An unspaced CJK token had more than one vocabulary-supported - #: surname split ("夏侯惇": 夏侯 + 惇 was taken, 夏 + 侯惇 also - #: matched); the longest surname won, and ``detail`` names both - #: readings. Points at the two tokens the split produced. (#271) + #: A division of an unspaced CJK token that the parse had to + #: choose, from either of the two things that can divide one. + #: A VOCABULARY fork: more than one surname-supported split + #: existed ("夏侯惇" was taken as 夏侯 + 惇, while 夏 + 侯惇 also + #: matched), longest-match picked, and ``detail`` names both + #: readings (#271). Or a SEGMENTER answer scoring under the + #: stage's confidence floor: only one reading was offered, but the + #: score says it was a statistical guess rather than a stated + #: certainty, and ``detail`` names the pieces and the score + #: (#272). Either way it points at ALL the tokens the division + #: produced -- two for a vocabulary split, n+1 for a segmenter + #: answer cutting n times. SEGMENTATION = "segmentation" diff --git a/nameparser/locales/ja.py b/nameparser/locales/ja.py index 8ee21da..a0f9c26 100644 --- a/nameparser/locales/ja.py +++ b/nameparser/locales/ja.py @@ -80,12 +80,22 @@ # repertoire, because a katakana-bearing token can still be a # kana-licensed composite the pack acts on (山田エミ) -- DEVIATES must # declare it, and the adapter must not decline it. A pure-katakana -# token never reaches either: the stage's own gate stops it. +# token never reaches THE ADAPTER: KATAKANA is in no activation set, so +# the stage's own gate stops it before the segmenter is consulted. +# DEVIATES still declares one, and must: it is a predicate over any +# name at all, called before any gate runs, and over-declaring is its +# safe direction by design. _JA_RANGES = ((0x3005, 0x3005), (0x3040, 0x309F), (0x30A0, 0x30FF), (0x3400, 0x4DBF), (0x4E00, 0x9FFF), (0xF900, 0xFAFF), (0x20000, 0x323AF)) +#: How far outside [0, 1] a namedivider score may stray and still count +#: as float noise around a real answer rather than a broken divider -- +#: see the adapter's own comment for why it is float32-sized. +_SCORE_EPSILON = 1e-6 + + def _in_repertoire(char: str) -> bool: return any(lo <= ord(char) <= hi for lo, hi in _JA_RANGES) @@ -118,7 +128,10 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: The adapter declines -- returns None, leaving the token whole -- for text outside the Japanese repertoire, for text too short to - divide, and for any answer that fails to reconstruct its input. An + divide, for any answer that fails to reconstruct its input, and for + any score outside [0, 1] by more than float noise (a divider + scoring 87.0 is broken, and its division is worth no more than its + score). An answer that puts the whole token on one side comes back as "confidently one token" rather than a decline: the divider read it and found nothing to cut, which is not the same as having no @@ -126,10 +139,16 @@ def ja_segmenter(*, gbdt: bool = False) -> Segmenter: Answers arrive with namedivider's own confidence, which the parsing stage compares against its floor to decide whether to report a SEGMENTATION ambiguity: namedivider scores a rule-based division - (the kana boundary in 高橋みなみ, a two-character name) 1.0 and a - kanji-statistics division well below that, so in practice every - statistically-divided name carries the report and every - rule-divided one does not. + (the kana boundary in 高橋みなみ, or a two-character name) 1.0 and + a kanji-statistics division well below that, so every + statistically divided name carries the report and every + rule-divided one carries none. The two-character rule is the case + to know about: dividing 原恵 one character each way is a + presumption Japanese practice makes, not a measurement (usage.rst + says so), yet the 1.0 keeps it silent. That is namedivider's own + stated certainty, taken as given here; ``DividedName.algorithm`` + names which rule answered and is the hook if presumption-reporting + is ever wanted. """ try: import namedivider @@ -158,11 +177,33 @@ def segment(text: str) -> Segmentation | None: result = divider.divide_name(text) if result.family + result.given != text: return None # defensive: never trust reconstruction - # float(): the score is a numpy scalar on the statistical path, - # and clamping guards the [0, 1] range Segmentation enforces - # against float error at the edge of namedivider's softmax (a - # violation would raise mid-parse). - confidence = min(1.0, max(0.0, float(result.score))) + # float(): the score is a numpy scalar on the statistical path. + # Two different failures wear the same shape here, and they get + # different answers. Float error at the edge of namedivider's + # softmax is arithmetic noise around a real answer, so an + # epsilon either side of [0, 1] is tolerated and clamped -- + # Segmentation enforces the range and would otherwise raise + # mid-parse. A score OUTSIDE that epsilon is not noise: a + # divider reporting 87.0 confidence is broken, and its division + # is worth nothing either. Declining keeps the parse safe AND + # observable by absence -- an unconditional clamp mapped 87.0 + # to 1.0, i.e. "certain", silencing the SEGMENTATION report at + # exactly the point of maximum brokenness. NaN takes the same + # exit for free: every comparison against it is False, so it + # fails both bounds below and declines. + # + # _SCORE_EPSILON is sized for FLOAT32, not float64: numpy + # softmax output is commonly float32, whose eps is ~1.2e-7, and + # namedivider's rule-based answers score exactly 1.0 -- so a + # float64-sized 1e-9 band would decline a correct rule-based + # division that came back a single ulp high (1.0000001 is + # exactly that). A few float32 ulps is still five orders of + # magnitude below any score a broken divider would produce, so + # nothing is bought by tightening it. + score = float(result.score) + if not -_SCORE_EPSILON <= score <= 1.0 + _SCORE_EPSILON: + return None + confidence = min(1.0, max(0.0, score)) # Defensive, the reconstruction check's twin: no namedivider # 0.4.x path can return an empty side (every answer, rule or # statistical, is built by slicing the input at an interior diff --git a/tests/v2/cases.py b/tests/v2/cases.py index c8b6e44..db0d77a 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -617,8 +617,10 @@ def __post_init__(self) -> None: {"family": "佐々木", "given": "太郎"}, classification="fix(#272)", notes="々 (U+3005, the ideographic iteration mark) repeats " - "the preceding kanji and is Script=Common under UAX " - "#24; it counts as Han here, or 佐々木 -- a top-20 " + "the preceding kanji and is Script=Han under UAX #24, " + "but sits outside every CJK ideograph BLOCK -- and the " + "classifier is a block table, so it needs a singleton " + "entry to count as Han; without one 佐々木 -- a top-20 " "Japanese surname -- would be a mixed-script token and " "reverse"), Case("ja_nakaguro_inside_a_nickname", diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index d933d1e..73b4718 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -248,8 +248,9 @@ def test_segmenter_decline_and_confident_whole_are_distinct() -> None: # "confidently one token" -- with the same effect on the tokens: # the difference is the protocol's to carry, not this stage's for seg in (_declines, _fake(())): - assert _texts(_run("山田太郎", policy=_JA, segmenter=seg)) == \ - ["山田太郎"] + out = _run("山田太郎", policy=_JA, segmenter=seg) + assert _texts(out) == ["山田太郎"] + assert out.ambiguities == () # nothing decided, nothing to report def test_kana_licensed_token_gates_in_for_the_segmenter() -> None: @@ -264,16 +265,18 @@ def test_pure_katakana_never_gates_in() -> None: # the license's other half: a lone katakana token is predominantly # a transcribed foreign name, so KATAKANA sits in no activation set # and no segmenter is consulted for one - assert _texts(_run("マイケル", policy=_JA, segmenter=_fake((2,)))) \ - == ["マイケル"] + out = _run("マイケル", policy=_JA, segmenter=_fake((2,))) + assert _texts(out) == ["マイケル"] + assert out.ambiguities == () def test_kana_licensed_token_stays_whole_under_default_policy() -> None: # the pack IS the language declaration: without it the default # activation is HANGUL alone, so nothing gates in and the # configured segmenter is inert - assert _texts(_run("高橋みなみ", segmenter=_fake((2,)))) == \ - ["高橋みなみ"] + out = _run("高橋みなみ", segmenter=_fake((2,))) + assert _texts(out) == ["高橋みなみ"] + assert out.ambiguities == () def test_segmenter_multi_split_produces_n_plus_one_tokens() -> None: @@ -319,16 +322,22 @@ def test_high_confidence_and_no_confidence_emit_nothing() -> None: assert state.ambiguities == () -def test_out_of_bounds_split_is_a_decline() -> None: +def test_out_of_bounds_split_raises() -> None: # Segmentation cannot check the upper bound (it never sees the - # text), so the stage does -- a violating answer is a buggy - # segmenter, and a buggy segmenter must not crash a parse - assert _texts(_run("山田", policy=_JA, segmenter=_fake((5,)))) == \ - ["山田"] + # text), so the stage does -- and it RAISES, the same call the + # wrong-answer-type check beside it makes: both are protocol + # violations by the segmenter's author, inside the declared + # totality exception. Declining silently instead (as this did + # before the review) made an off-by-one segmenter undebuggable -- + # every answer vanished and the name merely looked undivided. + with pytest.raises(ValueError, + match=r"segmenter returned splits beyond the " + r"token: last offset 5, token length 2"): + _run("山田", policy=_JA, segmenter=_fake((5,))) # 2 on a two-character token is the boundary itself: the check is # >=, not >, since a cut at len(text) would leave an empty piece - assert _texts(_run("山田", policy=_JA, segmenter=_fake((2,)))) == \ - ["山田"] + with pytest.raises(ValueError, match="last offset 2, token length 2"): + _run("山田", policy=_JA, segmenter=_fake((2,))) def test_family_comma_gates_the_segmenter_too() -> None: @@ -367,3 +376,36 @@ def seg(text: str) -> Segmentation | None: _run("山田太郎", policy=_JA, segmenter=seg) +def test_a_neighbour_in_an_UNACTIVATED_script_still_blocks_the_consult() -> None: + # The precondition counts a second script-written token whatever + # script it is written in -- effective_script merely non-None, NOT + # membership in the activation set. A katakana or hangul neighbour + # is a boundary its writer drew just as deliberately as a Han one, + # so the name is already divided and the segmenter must not be + # asked. Narrowing the check to `in scripts` would make both of + # these split 山 + 田太郎 while the writer's own boundary sat one + # token to the right. + for name in ("山田太郎 マイケル", "山田太郎 김민준"): + out = _run(name, policy=_JA, segmenter=_fake((1,))) + assert _texts(out) == name.split(), name + assert out.ambiguities == (), name + + +def test_the_segmenter_is_handed_the_gated_token_only() -> None: + # It is asked where ONE token divides, so it gets that token's text + # and nothing else -- not the original, not the name part joined + # back together. A leading Latin title is the case that can tell + # the difference: it is skipped by the gate (no script) and is not + # a neighbour for the precondition either, so the consult happens + # and the argument is observable. + asked: list[str] = [] + + def seg(text: str) -> Segmentation | None: + asked.append(text) + return Segmentation((2,)) + + out = _run("Dr 山田太郎", policy=_JA, segmenter=seg) + assert asked == ["山田太郎"] + assert _texts(out) == ["Dr", "山田", "太郎"] + + diff --git a/tests/v2/pipeline/test_vocab.py b/tests/v2/pipeline/test_vocab.py index d9498c5..1ca664e 100644 --- a/tests/v2/pipeline/test_vocab.py +++ b/tests/v2/pipeline/test_vocab.py @@ -173,3 +173,25 @@ def test_nfd_hangul_still_classifies() -> None: nfd = unicodedata.normalize("NFD", "김민준") assert nfd != "김민준" # sanity: confirms the decomposition actually ran assert single_script(nfd) is Script.HANGUL + + +def test_script_ranges_are_pairwise_disjoint() -> None: + # The table's own comment states this invariant and nothing + # checked it: single_script returns the FIRST covering entry (dict + # iteration order), so any overlap would make the answer depend on + # declaration order instead of being well-defined. Compared over + # every entry of every script rather than script by script, + # because the adjacent blocks are exactly where a future edit + # would go wrong -- HIRAGANA ends at U+309F and KATAKANA opens at + # U+30A0 with no gap at all, and HAN's singleton U+3005 sits just + # below both. + spans = [(lo, hi, script) + for script, ranges in _SCRIPT_RANGES.items() + for lo, hi in ranges] + assert len(spans) > 1 # never passes vacuously + for i, (lo, hi, script) in enumerate(spans): + assert lo <= hi, f"{script} range ({lo:#x}, {hi:#x}) is inverted" + for other_lo, other_hi, other in spans[i + 1:]: + assert hi < other_lo or other_hi < lo, ( + f"{script} range ({lo:#x}, {hi:#x}) overlaps {other} " + f"range ({other_lo:#x}, {other_hi:#x})") diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 5b4811f..a75530b 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -60,3 +60,16 @@ def test_cli_locale_empty_string_errors_not_silent_default() -> None: proc = _run("John Smith", "--locale", "") assert proc.returncode != 0 assert "ru" in proc.stderr and "tr_az" in proc.stderr + + +def test_cli_locale_ja_activates_nothing_by_itself() -> None: + # usage.rst's claim, pinned at the layer it is about: the CLI takes + # the pack but has no way to attach a segmenter, and the ja pack + # ships no vocabulary -- so --locale ja is byte-identical to the + # default parser on the very name the pack exists to divide. A + # future --segmenter flag, or a pack that grew a surname list, + # would land here first. + packed = _run("山田太郎", "--locale", "ja") + default = _run("山田太郎") + assert packed.returncode == 0 + assert packed.stdout == default.stdout diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index acc6f71..9796eae 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -270,7 +270,7 @@ def test_ja_adapter_guard_stack_against_a_stub( def test_ja_adapter_defensive_branches_against_a_stub( monkeypatch: pytest.MonkeyPatch) -> None: # reconstruction failure -> decline; an empty side -> the stated - # "confidently undivided" opinion; an out-of-range score -> clamped + # "confidently undivided" opinion; a score a hair over 1 -> clamped, # so Segmentation's [0, 1] validation cannot raise mid-parse _fake_namedivider( monkeypatch, lambda text: _FakeDivided("外", "れ", 0.5)) @@ -282,6 +282,39 @@ def test_ja_adapter_defensive_branches_against_a_stub( assert whole.splits == () and whole.confidence == 1.0 +def test_ja_adapter_declines_a_garbage_score( + monkeypatch: pytest.MonkeyPatch) -> None: + # The other half of the score handling, and the half the old + # unconditional clamp got wrong: float noise at the edge of a + # softmax is arithmetic around a real answer and clamps (above), + # but 87.0 is not noise -- a divider scoring that is broken, and + # its DIVISION is worth no more than its score. Clamping mapped it + # to 1.0, i.e. "certain", which silenced the SEGMENTATION report at + # exactly the point of maximum brokenness. Declining leaves the + # token whole: safe, and observable by the report's absence. + _fake_namedivider( + monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], 87.0)) + assert locales.ja_segmenter()("山田太郎") is None + _fake_namedivider( + monkeypatch, lambda text: _FakeDivided(text[:2], text[2:], -0.5)) + assert locales.ja_segmenter()("山田太郎") is None + + +def test_ja_adapter_declines_a_nan_score( + monkeypatch: pytest.MonkeyPatch) -> None: + # NaN takes the garbage-score exit for free rather than needing one + # of its own: every comparison against NaN is False, so it fails + # BOTH bounds of the epsilon band. Worth a test precisely because + # it is a property of the comparison rather than a written branch + # -- under the old clamp, min/max propagated it straight into + # Segmentation, whose own range check would then have raised + # mid-parse. + _fake_namedivider( + monkeypatch, + lambda text: _FakeDivided(text[:2], text[2:], float("nan"))) + assert locales.ja_segmenter()("山田太郎") is None + + def test_ja_adapter_gbdt_flag_selects_the_gbdt_divider( monkeypatch: pytest.MonkeyPatch) -> None: divide = _fake_namedivider( @@ -321,10 +354,12 @@ def test_ja_end_to_end() -> None: # kana-licensed composite: gated in through the HIRAGANA entry n = p.parse("高橋みなみ") assert (n.family, n.given) == ("高橋", "みなみ") - # the iteration mark 々 gates in as Han and namedivider reads the + # the iteration mark 々 gates in as Han (Script=Han under UAX #24, + # but outside the ideograph blocks the classifier spans, so it + # rides in on a singleton range entry) and namedivider reads the # repeated character correctly -- verified against 0.4.1 rather - # than assumed, since 々 is Script=Common and a divider could - # plausibly have choked on it or cut beside it + # than assumed, since a divider could plausibly have choked on it + # or cut beside it n = p.parse("佐々木健") assert (n.family, n.given) == ("佐々木", "健") @@ -406,6 +441,57 @@ def test_ja_reports_statistical_divisions_and_not_rule_based_ones() -> None: assert p.parse("原恵").ambiguities == () # two-character rule +@_needs_ja +def test_ja_divides_a_lone_hiragana_token() -> None: + # The whole-name presumption, applied to hiragana: at token level a + # bare given name is indistinguishable from a full name, so the + # stage gates みなみ in and namedivider divides it -- み + なみ, + # which is wrong if the writer meant the given name alone and right + # if they meant a (rare) all-kana full name. Exactly the acceptance + # 田中 -> 田 + 中 already carries for kanji, pinned here so the kana + # path's version is a recorded consequence rather than a surprise. + # The statistical score puts a SEGMENTATION report on it, which is + # the caller's handle for routing it to review. + n = _PACKED["ja"].parse("みなみ") + assert (n.family, n.given) == ("み", "なみ") + assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] + + +@_needs_ja +def test_ja_divides_an_astral_kanji_name() -> None: + # 𠮷 (U+20BB7, the "tsuchiyoshi" 吉) is a real surname character in + # the supplementary plane -- the reason _SCRIPT_RANGES carries the + # astral Han span at all. It has to survive the whole chain as one + # character and not two: the script gate, the adapter's repertoire + # scan, namedivider itself, and the offset arithmetic that turns + # the answer into spans (Python indexes by codepoint, so an offset + # of 2 here spans four UTF-16 units -- a surrogate-pair bug would + # land the split in the middle of the character). + n = _PACKED["ja"].parse("𠮷田太郎") + assert (n.family, n.given) == ("𠮷田", "太郎") + assert [a.kind for a in n.ambiguities] == [AmbiguityKind.SEGMENTATION] + + +@_needs_ja +def test_ja_stacked_on_zh_pays_the_zh_packs_documented_cost() -> None: + # Stacking composes MECHANICALLY (the test above) but not for free, + # and this pins the cost so it stays a documented trade rather than + # a bug report. Vocabulary runs first, so a Japanese kanji name + # opening on a listed CHINESE surname never reaches the segmenter: + # 高 is a common Chinese surname, so 高橋一郎 splits 高 + 橋一郎 + # under ZH+JA exactly as it does under ZH alone -- the same reading + # cases.py's zh_japanese_kanji_tradeoff row records, and the reason + # the docs call the two packs corpus alternatives and qualify the + # stack (see _script_segment.py's module docstring, usage.rst and + # the release log). JA alone reads it correctly (test_ja_end_to_end). + p = parser_for(locales.ZH, locales.JA, segmenter=locales.ja_segmenter()) + n = p.parse("高橋一郎") + assert (n.family, n.given) == ("高", "橋一郎") + # and silently: one vocabulary match is not a fork, so nothing here + # flags the mis-split -- the pack choice is what has to be right + assert n.ambiguities == () + + def test_pack_vocabulary_entries_are_single_words() -> None: # The shipped-vocabulary guard (test_lexicon.py's # test_default_lexicon_builds_warning_free), extended to the OTHER diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index a6902d0..656659c 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -484,11 +484,13 @@ def test_kana_licensed_names_read_family_first_by_default() -> None: def test_iteration_mark_counts_as_han() -> None: - # 々 (U+3005) repeats the preceding kanji and is Script=Common - # under UAX #24, so classifying by the Script property would put a - # 佐々木 token in no script at all: the name reversed, and the - # token never reached the segmentation gate. It is in the HAN - # ranges by BLOCK-style reasoning, the same call U+30FB gets. + # 々 (U+3005) repeats the preceding kanji. It is Script=Han under + # UAX #24 already, but it lives outside every CJK ideograph BLOCK, + # and the classifier is a block table -- so without its singleton + # entry a 佐々木 token was in no script at all: the name reversed, + # and the token never reached the segmentation gate. The Script + # property would have got this one right unaided; the block table + # is what needed the special case. n = parse("佐々木 太郎") assert (n.family, n.given) == ("佐々木", "太郎") n = parse("野々村 真") @@ -659,6 +661,23 @@ def explicit(text: str) -> Segmentation | None: assert p.segmenter is explicit +def test_parser_for_segmenter_none_clears_the_base() -> None: + # the third state UNSET buys (#272 review): None is a VALUE here, + # not an absence, so passing it explicitly drops the base's + # segmenter instead of inheriting the very thing it was asked to + # remove. Omitting the keyword is what carries the base's through + # (the test above), and this is how you derive an unsegmented + # parser from a segmented one without rebuilding its lexicon and + # policy by hand. + def seg(text: str) -> Segmentation | None: + return None + base = Parser(segmenter=seg) + pack = locales.get("zh") + assert parser_for(pack, base=base, segmenter=None).segmenter is None + # ...and the base is untouched: parser_for builds a fresh Parser + assert base.segmenter is seg + + def test_parser_picklability_is_conditional_on_the_segmenter() -> None: # declared contract (locales spec section 4): Parser pickles iff # its segmenter does -- like any callable-holding object diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index aad8ad9..8a0abd9 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -100,6 +100,24 @@ def test_segmentation_rejects_mappings_and_non_iterables() -> None: Segmentation({2: "a", 3: "b"}) # type: ignore[arg-type] with pytest.raises(TypeError, match="Segmentation.splits"): Segmentation(5) # type: ignore[arg-type] + # ...and the bare string, the third member of that family and the + # one with teeth: "" is iterable and empty, so without the guard + # Segmentation("") became "confidently one token" -- an opinion + # nobody stated -- while "23" failed one character deep naming '2' + # instead of the argument. Both must name the field. + for bad in ("", "23"): + with pytest.raises(TypeError, match="Segmentation.splits"): + Segmentation(bad) # type: ignore[arg-type] + + +def test_segmentation_rejects_bools_in_both_fields() -> None: + # bool is an int subclass in both slots: True as an offset is a + # comparison result leaking into an index, True as a confidence one + # leaking into a score. Behavioral twin of Token's span guard. + with pytest.raises(TypeError, match="Segmentation.splits"): + Segmentation((True,)) # type: ignore[list-item] + with pytest.raises(TypeError, match="Segmentation.confidence"): + Segmentation((2,), confidence=True) # type: ignore[arg-type] def test_segmentation_lets_a_generator_raise_its_own_error() -> None: @@ -419,6 +437,11 @@ def test_types_pickle_round_trip() -> None: assert pickle.loads(pickle.dumps(amb)) == amb tok = Token("de", Span(9, 11), Role.FAMILY, frozenset({"particle"})) assert pickle.loads(pickle.dumps(tok)) == tok + # the family sweep is the point of this test, so a frozen type + # added later belongs in it -- Segmentation (#272) carries the same + # guarded __getstate__/__setstate__ pair as its three siblings + seg = Segmentation((2,), confidence=0.42) + assert pickle.loads(pickle.dumps(seg)) == seg def test_span_add_is_blocked() -> None: From 332bd7a6ec1c7b1bd3ae2c9069bfee6fb3c2d055 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 29 Jul 2026 17:56:36 -0700 Subject: [PATCH 14/14] Apply the simplify pass (#272) Deduped findings of a four-angle /simplify review, no behavior changes: * _vocab: extract the shared _classify(normalized) step, so effective_script normalizes ONCE instead of twice -- deleting the comment block that defended the second normalize. * _script_segment: the two-character-presumption paragraph was written at near-identical length here and in locales/ja.py; the stage keeps one sentence and points at the pack that knows namedivider's rules. * _script_segment: "one split path so nothing drifts" was stated three times (module docstring, _pieces, _split); it stays on _split, the shared path itself. * _script_segment: the upper-bound ownership fact was at three sites; _split's precondition keeps one clause, the raise site and Segmentation's docstring keep the argument. * tests: six `return None` segmenter callables collapse to the pickle test's module-level one plus two named siblings -- two, because the override test needs distinct objects for its identity assertion. * tests: the capture-the-argument closure was written twice; both cases now share a _capture() helper beside _fake. * tests: normalize two files' triple trailing newline to one. * AGENTS.md: promote the raise-vs-decline segmenter doctrine out of the Pickling bullet into its own, leaving Pickling one clause. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 3 +- nameparser/_pipeline/_script_segment.py | 32 ++++++------------ nameparser/_pipeline/_vocab.py | 43 ++++++++++++------------ tests/v2/pipeline/test_script_segment.py | 29 ++++++++-------- tests/v2/test_parser.py | 40 +++++++++++----------- tests/v2/test_types.py | 2 -- 6 files changed, 69 insertions(+), 80 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d34665d..29633d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -131,7 +131,8 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. -- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is the optional `Parser(segmenter=...)` hook — a Parser holding a callable pickles iff that callable does, and the same hook is parse-totality's one exception. Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. +- **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. - **Tests**: all v2 tests live in the `tests/v2/` package (its `conftest.py` overrides the v1 dual-run fixture — v2 code never reads shared `CONSTANTS`), one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, and `test_regex_sync.py`, which pins every hand-copied pattern or codepoint table against its source wherever the copy lives — including copies outside the package), names stating behavior. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/nameparser/_pipeline/_script_segment.py b/nameparser/_pipeline/_script_segment.py index 84edeb2..0365a76 100644 --- a/nameparser/_pipeline/_script_segment.py +++ b/nameparser/_pipeline/_script_segment.py @@ -30,10 +30,9 @@ packs are corpus ALTERNATIVES, one per corpus; stacking them is for genuinely mixed data that accepts that trade. Its Segmentation may cut anywhere and any number of times, which is why the split path -below is written for n cuts and the vocabulary hit is simply its -one-cut case. One precondition guards it that the vocabulary has no -twin of: a segmenter answers where an UNDIVIDED name divides, so it is -consulted only when the gated token is the name part's ONLY +below takes n cuts. One precondition guards it that the vocabulary +has no twin of: a segmenter answers where an UNDIVIDED name divides, +so it is consulted only when the gated token is the name part's ONLY script-written one -- "山田 太郎" was divided by its writer and must not have its family divided again (a Latin title or suffix draws no such boundary). A segmenter's own exceptions PROPAGATE -- the single @@ -109,14 +108,9 @@ #: STATISTICALLY divided name carries a SEGMENTATION report, and every #: RULE-divided one -- the kana boundary, a two-character name, #: namedivider's specific-name rules -- carries none. -#: One case is worth writing down rather than rediscovering: the -#: two-character division is a PRESUMPTION, not a measurement (usage.rst -#: says exactly that), yet namedivider scores it 1.0 and this floor -#: therefore keeps it silent. That is the divider's own stated -#: certainty, accepted here as-is rather than second-guessed by a -#: length rule of ours. If presumption-reporting is ever wanted, the -#: hook is namedivider's DividedName.algorithm, which names WHICH rule -#: answered -- not the score, which cannot tell the two apart. +#: namedivider scores its own two-character rule 1.0, so this floor +#: keeps that division silent -- see locales/ja.py for why the +#: presumption is accepted as stated. #: Not configurable in 2.x (YAGNI). _SEGMENTER_CONFIDENCE_FLOOR = 0.9 @@ -136,9 +130,7 @@ def _remap(run: tuple[int, ...], split_at: int, def _pieces(text: str, splits: tuple[int, ...]) -> tuple[str, ...]: - """`text` cut at every offset in `splits`: n offsets, n+1 pieces. - One idiom for both callers -- the split itself and the report that - describes it must never disagree about what the cuts produce.""" + """`text` cut at every offset in `splits`: n offsets, n+1 pieces.""" cuts = (0, *splits, len(text)) return tuple(text[a:b] for a, b in zip(cuts, cuts[1:])) @@ -149,13 +141,11 @@ def _split(state: ParseState, i: int, splits: tuple[int, ...], a SEGMENTATION report when there is one. The offsets arrive non-empty, ascending and interior whatever chose - them. From a segmenter: non-empty is the caller's own check, and + them. From a segmenter: non-empty is the caller's own check, strictly ascending with each >= 1 is Segmentation.__post_init__'s, - while the last offset being < len(text) is the caller's bounds - check -- that half cannot live in Segmentation, which never sees - the text. From the vocabulary: the single offset is >= 1 and - < len(text) by the range(cap, 0, -1) construction and its len-1 - cap. + and the last offset is < len(text) -- checked by the caller. From + the vocabulary: the single offset is >= 1 and < len(text) by the + range(cap, 0, -1) construction and its len-1 cap. The ONE split path: the vocabulary hit is the single-offset case and the segmenter's answer the general one, so neither can drift diff --git a/nameparser/_pipeline/_vocab.py b/nameparser/_pipeline/_vocab.py index 3c51c69..6638d8d 100644 --- a/nameparser/_pipeline/_vocab.py +++ b/nameparser/_pipeline/_vocab.py @@ -249,6 +249,16 @@ def _normalized_for_script(text: str) -> str | None: return unicodedata.normalize("NFC", text) +def _classify(normalized: str) -> Script | None: + """The FIRST _SCRIPT_PATTERNS entry covering all of `normalized` + (already NFC, via _normalized_for_script), else None. Shared by + both public classifiers so each of them normalizes exactly once.""" + for script, pattern in _SCRIPT_PATTERNS.items(): + if pattern.fullmatch(normalized): + return script + return None + + def single_script(text: str) -> Script | None: """The one Script whose ranges cover EVERY char of `text`, else None (mixed-script text has no well-defined convention to apply; @@ -259,10 +269,7 @@ def single_script(text: str) -> Script | None: normalized = _normalized_for_script(text) if normalized is None: return None - for script, pattern in _SCRIPT_PATTERNS.items(): - if pattern.fullmatch(normalized): - return script - return None + return _classify(normalized) def effective_script(text: str) -> Script | None: @@ -275,26 +282,18 @@ def effective_script(text: str) -> Script | None: HIRAGANA carrier entry. Pure-katakana stays KATAKANA (single_script's answer): a lone katakana token is predominantly a transcribed foreign name, so nothing defaults on it.""" - script = single_script(text) + # None for both shapes _JA_PATTERN could never match anyway (empty + # text, or all-ASCII text): real work, not a leftover "if text" + # guard, since the ASCII case is one a bare emptiness check would + # let through. The single normalized copy then serves both the + # single-script answer and the license below. + normalized = _normalized_for_script(text) + if normalized is None: + return None + script = _classify(normalized) if script is not None: return script - # Renormalizes text that single_script (above) already normalized - # once: deliberately NOT hoisted into a shared `_classify(normalized)` - # helper. The second call only runs on the fall-through path (a - # token single_script could not classify at all, i.e. genuinely - # mixed-script or empty/ASCII), never on the common single-script - # hit above, so it is a quick re-check on already-NFC text, not - # measured work -- the per-char-vs-regex history in this module's - # header is what a real cost here would look like, and this isn't - # it. - # normalized is None for both shapes _JA_PATTERN could never match - # anyway (empty text, or the all-ASCII text single_script's fast - # path already ruled out) -- real work, not a leftover "if text" - # guard: unlike the pre-NFC version, None here also covers the - # ASCII case, which single_script's own empty check alone would - # not. - normalized = _normalized_for_script(text) - if normalized is not None and _JA_PATTERN.fullmatch(normalized): + if _JA_PATTERN.fullmatch(normalized): return Script.HIRAGANA return None diff --git a/tests/v2/pipeline/test_script_segment.py b/tests/v2/pipeline/test_script_segment.py index 73b4718..bb14d77 100644 --- a/tests/v2/pipeline/test_script_segment.py +++ b/tests/v2/pipeline/test_script_segment.py @@ -49,6 +49,19 @@ def segmenter(text: str) -> Segmentation | None: return segmenter +def _capture( + splits: tuple[int, ...] = (2,)) -> tuple[list[str], Segmenter]: + """_fake, plus the list of texts it was handed: the two cases below + that pin WHETHER and WITH WHAT the stage consults a segmenter need + the same closure.""" + asked: list[str] = [] + + def segmenter(text: str) -> Segmentation | None: + asked.append(text) + return Segmentation(splits) + return asked, segmenter + + def _declines(text: str) -> Segmentation | None: return None @@ -232,12 +245,7 @@ def test_vocabulary_wins_over_the_segmenter() -> None: # precedence, and the reason parser_for(ZH, JA, segmenter=...) # composes: a matched surname is a dictionary certainty, so the # segmenter is not even asked - asked: list[str] = [] - - def seg(text: str) -> Segmentation | None: - asked.append(text) - return Segmentation((2,)) - + asked, seg = _capture() state = _run("毛泽东", policy=_JA, segmenter=seg) assert _texts(state) == ["毛", "泽东"] assert asked == [] @@ -398,14 +406,7 @@ def test_the_segmenter_is_handed_the_gated_token_only() -> None: # the difference: it is skipped by the gate (no script) and is not # a neighbour for the precondition either, so the consult happens # and the argument is observable. - asked: list[str] = [] - - def seg(text: str) -> Segmentation | None: - asked.append(text) - return Segmentation((2,)) - + asked, seg = _capture() out = _run("Dr 山田太郎", policy=_JA, segmenter=seg) assert asked == ["山田太郎"] assert _texts(out) == ["Dr", "山田", "太郎"] - - diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 656659c..08c64fa 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -620,6 +620,17 @@ def _module_level_decline(text: str) -> Segmentation | None: return None # module-level so pickle can find it +# Inert sentinels for the plumbing tests below, which only ever compare +# identity. Two of them, because the override test needs the loser and +# the winner to be DISTINCT objects or its assertion is vacuous. +def _decline_a(text: str) -> Segmentation | None: + return None + + +def _decline_b(text: str) -> Segmentation | None: + return None + + def test_parser_segmenter_is_keyword_only_and_validated() -> None: assert Parser().segmenter is None with pytest.raises(TypeError, match="callable"): @@ -629,11 +640,9 @@ def test_parser_segmenter_is_keyword_only_and_validated() -> None: def test_parser_for_carries_the_base_segmenter() -> None: - def seg(text: str) -> Segmentation | None: - return None # not given: the base's carries through unchanged - p = parser_for(locales.get("zh"), base=Parser(segmenter=seg)) - assert p.segmenter is seg + p = parser_for(locales.get("zh"), base=Parser(segmenter=_decline_a)) + assert p.segmenter is _decline_a def test_parser_for_rejects_a_non_parser_base() -> None: @@ -642,23 +651,16 @@ def test_parser_for_rejects_a_non_parser_base() -> None: def test_parser_for_takes_a_segmenter_keyword() -> None: - def seg(text: str) -> Segmentation | None: - return None - p = parser_for(locales.get("zh"), segmenter=seg) - assert p.segmenter is seg + p = parser_for(locales.get("zh"), segmenter=_decline_a) + assert p.segmenter is _decline_a assert parser_for(locales.get("zh")).segmenter is None def test_parser_for_segmenter_keyword_overrides_the_base() -> None: # later wins, the same rule scalar policy fields follow - def from_base(text: str) -> Segmentation | None: - return None - - def explicit(text: str) -> Segmentation | None: - return None - p = parser_for(locales.get("zh"), base=Parser(segmenter=from_base), - segmenter=explicit) - assert p.segmenter is explicit + p = parser_for(locales.get("zh"), base=Parser(segmenter=_decline_a), + segmenter=_decline_b) + assert p.segmenter is _decline_b def test_parser_for_segmenter_none_clears_the_base() -> None: @@ -669,13 +671,11 @@ def test_parser_for_segmenter_none_clears_the_base() -> None: # (the test above), and this is how you derive an unsegmented # parser from a segmented one without rebuilding its lexicon and # policy by hand. - def seg(text: str) -> Segmentation | None: - return None - base = Parser(segmenter=seg) + base = Parser(segmenter=_decline_a) pack = locales.get("zh") assert parser_for(pack, base=base, segmenter=None).segmenter is None # ...and the base is untouched: parser_for builds a fresh Parser - assert base.segmenter is seg + assert base.segmenter is _decline_a def test_parser_picklability_is_conditional_on_the_segmenter() -> None: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 8a0abd9..a0478ee 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -528,5 +528,3 @@ def test_with_field_tokens_rejects_mismatched_roles() -> None: with pytest.raises(ValueError, match="has role family, not given"): name._with_field_tokens( {Role.GIVEN: (Token("x", None, Role.FAMILY),)}) - -