From fd0096a1a22201ec58b1b54b3a2bda3a5417f6e4 Mon Sep 17 00:00:00 2001 From: leycec Date: Fri, 19 Apr 2024 03:04:35 -0400 Subject: [PATCH] `dict[tuple[...], ...]` type hints. This commit resolves a critical low-level issue in @beartype's dynamic type-checking code generator for **nested tuple-in-dictionary type hints** (i.e., type hints of the form `dict[tuple[...], ...]`), resolving issue #371 kindly submitted by that wascally typing wabbit @alisaifee (Ali-Akber Saifee). The accursed @beartype 0.18.X release cycle continues to bedevil our poor world. (*Flattened madness is no glad flattery!*) --- beartype/_check/code/codemagic.py | 242 +++++++++--------- beartype/_check/code/codemake.py | 79 ++++-- beartype/_check/code/snip/codesnipcls.py | 12 +- .../a00_code/snip/test_codesnipcls.py | 4 +- .../data/hint/pep/proposal/_data_pep585.py | 36 ++- .../data/hint/pep/proposal/data_pep484.py | 36 ++- 6 files changed, 248 insertions(+), 161 deletions(-) diff --git a/beartype/_check/code/codemagic.py b/beartype/_check/code/codemagic.py index 40f60551..c3ffad0d 100644 --- a/beartype/_check/code/codemagic.py +++ b/beartype/_check/code/codemagic.py @@ -31,123 +31,125 @@ exception messages. ''' -# ....................{ HINT ~ meta }.................... -# Iterator yielding the next integer incrementation starting at 0, to be safely -# deleted *AFTER* defining the following 0-based indices via this iterator. -__hint_meta_index_counter = count(start=0, step=1) - - -HINT_META_INDEX_HINT = next(__hint_meta_index_counter) -''' -0-based index into each tuple of hint metadata providing the currently -visited hint. - -For both space and time efficiency, this metadata is intentionally stored as -0-based integer indices of an unnamed tuple rather than: - -* Human-readable fields of a named tuple, which incurs space and time costs we - would rather *not* pay. -* 0-based integer indices of a tiny fixed list. Previously, this metadata was - actually stored as a fixed list. However, exhaustive profiling demonstrated - that reinitializing each such list by slice-assigning that list's items from - a tuple to be faster than individually assigning these items: - - .. code-block:: shell-session - - $ echo 'Slice w/ tuple:' && command python3 -m timeit -s \ - 'muh_list = ["a", "b", "c", "d",]' \ - 'muh_list[:] = ("e", "f", "g", "h",)' - Slice w/ tuple: - 2000000 loops, best of 5: 131 nsec per loop - $ echo 'Slice w/o tuple:' && command python3 -m timeit -s \ - 'muh_list = ["a", "b", "c", "d",]' \ - 'muh_list[:] = "e", "f", "g", "h"' - Slice w/o tuple: - 2000000 loops, best of 5: 138 nsec per loop - $ echo 'Separate:' && command python3 -m timeit -s \ - 'muh_list = ["a", "b", "c", "d",]' \ - 'muh_list[0] = "e" - muh_list[1] = "f" - muh_list[2] = "g" - muh_list[3] = "h"' - Separate: - 2000000 loops, best of 5: 199 nsec per loop - -So, not only does there exist no performance benefit to flattened fixed lists, -there exists demonstrable performance costs. -''' - - -HINT_META_INDEX_PLACEHOLDER = next(__hint_meta_index_counter) -''' -0-based index into each tuple of hint metadata providing the **current -placeholder type-checking substring** (i.e., placeholder to be globally -replaced by a Python code snippet type-checking the current pith expression -against the hint described by this metadata on visiting that hint). - -This substring provides indirection enabling the currently visited parent hint -to defer and delegate the generation of code type-checking each child argument -of that hint to the later time at which that child argument is visited. - -Example -------- -For example, the -:func:`beartype._decor._hint._pep._pephint.make_func_pith_code` function might -generate intermediary code resembling the following on visiting the -:data:`Union` parent of a ``Union[int, str]`` object *before* visiting either -the :class:`int` or :class:`str` children of that object: - - if not ( - @{0}! or - @{1}! - ): - raise get_func_pith_violation( - func=__beartype_func, - pith_name=$%PITH_ROOT_NAME/~, - pith_value=__beartype_pith_root, - ) - -Note the unique substrings ``"@{0}!"`` and ``"@{1}!"`` in that code, which that -function iteratively replaces with code type-checking each of the child -arguments of that :data:`Union` parent (i.e., :class:`int`, :class:`str`). The -final code memoized by that function might then resemble: - - if not ( - isinstance(__beartype_pith_root, int) or - isinstance(__beartype_pith_root, str) - ): - raise get_func_pith_violation( - func=__beartype_func, - pith_name=$%PITH_ROOT_NAME/~, - pith_value=__beartype_pith_root, - ) -''' - - -HINT_META_INDEX_PITH_EXPR = next(__hint_meta_index_counter) -''' -0-based index into each tuple of hint metadata providing the **current -pith expression** (i.e., Python code snippet evaluating to the current possibly -nested object of the passed parameter or return value to be type-checked -against the currently visited hint). -''' - - -HINT_META_INDEX_PITH_VAR_NAME = next(__hint_meta_index_counter) -''' -0-based index into each tuple of hint metadata providing the **current pith -variable name** (i.e., name of the unique local variable assigned the value of -the current pith either by a prior assignment statement or expression). -''' - - -HINT_META_INDEX_INDENT_LEVEL = next(__hint_meta_index_counter) -''' -0-based index into each tuple of hint metadata providing the **current -indentation level** (i.e., 1-based positive integer describing the current level -of indentation appropriate for the currently visited hint). -''' - - -# Delete the above counter for safety and sanity in equal measure. -del __hint_meta_index_counter +#FIXME: Preserved for documentation purposes. When time permits, centralized +#this documentation into the docstring of a new "HintMeta" dataclass, please. +# # ....................{ HINT ~ meta }.................... +# # Iterator yielding the next integer incrementation starting at 0, to be safely +# # deleted *AFTER* defining the following 0-based indices via this iterator. +# __hint_meta_index_counter = count(start=0, step=1) +# +# +# HINT_META_INDEX_HINT = next(__hint_meta_index_counter) +# ''' +# 0-based index into each tuple of hint metadata providing the currently +# visited hint. +# +# For both space and time efficiency, this metadata is intentionally stored as +# 0-based integer indices of an unnamed tuple rather than: +# +# * Human-readable fields of a named tuple, which incurs space and time costs we +# would rather *not* pay. +# * 0-based integer indices of a tiny fixed list. Previously, this metadata was +# actually stored as a fixed list. However, exhaustive profiling demonstrated +# that reinitializing each such list by slice-assigning that list's items from +# a tuple to be faster than individually assigning these items: +# +# .. code-block:: shell-session +# +# $ echo 'Slice w/ tuple:' && command python3 -m timeit -s \ +# 'muh_list = ["a", "b", "c", "d",]' \ +# 'muh_list[:] = ("e", "f", "g", "h",)' +# Slice w/ tuple: +# 2000000 loops, best of 5: 131 nsec per loop +# $ echo 'Slice w/o tuple:' && command python3 -m timeit -s \ +# 'muh_list = ["a", "b", "c", "d",]' \ +# 'muh_list[:] = "e", "f", "g", "h"' +# Slice w/o tuple: +# 2000000 loops, best of 5: 138 nsec per loop +# $ echo 'Separate:' && command python3 -m timeit -s \ +# 'muh_list = ["a", "b", "c", "d",]' \ +# 'muh_list[0] = "e" +# muh_list[1] = "f" +# muh_list[2] = "g" +# muh_list[3] = "h"' +# Separate: +# 2000000 loops, best of 5: 199 nsec per loop +# +# So, not only does there exist no performance benefit to flattened fixed lists, +# there exists demonstrable performance costs. +# ''' +# +# +# HINT_META_INDEX_PLACEHOLDER = next(__hint_meta_index_counter) +# ''' +# 0-based index into each tuple of hint metadata providing the **current +# placeholder type-checking substring** (i.e., placeholder to be globally +# replaced by a Python code snippet type-checking the current pith expression +# against the hint described by this metadata on visiting that hint). +# +# This substring provides indirection enabling the currently visited parent hint +# to defer and delegate the generation of code type-checking each child argument +# of that hint to the later time at which that child argument is visited. +# +# Example +# ------- +# For example, the +# :func:`beartype._decor._hint._pep._pephint.make_func_pith_code` function might +# generate intermediary code resembling the following on visiting the +# :data:`Union` parent of a ``Union[int, str]`` object *before* visiting either +# the :class:`int` or :class:`str` children of that object: +# +# if not ( +# @{0}! or +# @{1}! +# ): +# raise get_func_pith_violation( +# func=__beartype_func, +# pith_name=$%PITH_ROOT_NAME/~, +# pith_value=__beartype_pith_root, +# ) +# +# Note the unique substrings ``"@{0}!"`` and ``"@{1}!"`` in that code, which that +# function iteratively replaces with code type-checking each of the child +# arguments of that :data:`Union` parent (i.e., :class:`int`, :class:`str`). The +# final code memoized by that function might then resemble: +# +# if not ( +# isinstance(__beartype_pith_root, int) or +# isinstance(__beartype_pith_root, str) +# ): +# raise get_func_pith_violation( +# func=__beartype_func, +# pith_name=$%PITH_ROOT_NAME/~, +# pith_value=__beartype_pith_root, +# ) +# ''' +# +# +# HINT_META_INDEX_PITH_EXPR = next(__hint_meta_index_counter) +# ''' +# 0-based index into each tuple of hint metadata providing the **current +# pith expression** (i.e., Python code snippet evaluating to the current possibly +# nested object of the passed parameter or return value to be type-checked +# against the currently visited hint). +# ''' +# +# +# HINT_META_INDEX_PITH_VAR_NAME = next(__hint_meta_index_counter) +# ''' +# 0-based index into each tuple of hint metadata providing the **current pith +# variable name** (i.e., name of the unique local variable assigned the value of +# the current pith either by a prior assignment statement or expression). +# ''' +# +# +# HINT_META_INDEX_INDENT_LEVEL = next(__hint_meta_index_counter) +# ''' +# 0-based index into each tuple of hint metadata providing the **current +# indentation level** (i.e., 1-based positive integer describing the current level +# of indentation appropriate for the currently visited hint). +# ''' +# +# +# # Delete the above counter for safety and sanity in equal measure. +# del __hint_meta_index_counter diff --git a/beartype/_check/code/codemake.py b/beartype/_check/code/codemake.py index d6c1f4ae..13321fd1 100644 --- a/beartype/_check/code/codemake.py +++ b/beartype/_check/code/codemake.py @@ -31,11 +31,6 @@ from beartype._check.code.codemagic import ( EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL, EXCEPTION_PREFIX_HINT, - HINT_META_INDEX_HINT, - HINT_META_INDEX_PLACEHOLDER, - HINT_META_INDEX_PITH_EXPR, - HINT_META_INDEX_PITH_VAR_NAME, - HINT_META_INDEX_INDENT_LEVEL, ) from beartype._check.code.codescope import ( add_func_scope_type, @@ -540,7 +535,7 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: hint_child, hint_child_placeholder, pith_child_expr, - pith_curr_var_name, + pith_curr_var_name_index, indent_level_child, ) @@ -586,13 +581,36 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: f'index {hints_meta_index_curr} not tuple.' ) - #FIXME: [SPEED] Optimize by reducing to a single tuple unpacking. - # Localize metadatum for both efficiency and f-string purposes. - hint_curr = hint_curr_meta[HINT_META_INDEX_HINT] - hint_curr_placeholder = hint_curr_meta[HINT_META_INDEX_PLACEHOLDER] - pith_curr_expr = hint_curr_meta[HINT_META_INDEX_PITH_EXPR] - pith_curr_var_name = hint_curr_meta[HINT_META_INDEX_PITH_VAR_NAME] - indent_level_curr = hint_curr_meta[HINT_META_INDEX_INDENT_LEVEL] + #FIXME: ...heh. It's time, people. Sadly, it turns out that redefining + #the _enqueue_hint() closure on *EVERY* call to this function is a huge + #time sink -- far huger than anything else, actually. Therefore: + #* Define a new "HintMeta" dataclass defining one slotted field for each + # of these metadata. + #* Refactor the _enqueue_hint() closure into a HintMeta.enqueue_hint() + # method. + #* Replace all calls to the _enqueue_hint() closure with calls to the + # HintMeta.enqueue_hint() method. + #* Remove the _enqueue_hint() closure. + #* Remove all of the following locals from this function in favour of + # the "HintMeta" slotted fields of the same names: + # * hint_curr. + # * hint_curr_placeholder. + # * pith_curr_expr. + # * pith_curr_var_name_index. + # * indent_level_curr. + + # Localize metadata for both efficiency and f-string purposes. + # + # Note that list unpacking is substantially more efficient than + # manually indexing list items; the former requires only a single Python + # statement, whereas the latter requires "n" Python statements. + ( + hint_curr, + hint_curr_placeholder, + pith_curr_expr, + pith_curr_var_name_index, + indent_level_curr, + ) = hint_curr_meta # print(f'Visiting type hint {repr(hint_curr)}...') #FIXME: Comment this sanity check out after we're sufficiently @@ -749,7 +767,7 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: # was validated above to be supported. cls=get_hint_pep_origin_type_isinstanceable(hint_curr), func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL, ), ) # Else, this hint is either subscripted, not shallowly @@ -998,9 +1016,14 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: # >>> (a := b := "Mother's Illumination") # <-- not fine # SyntaxError: invalid syntax # - # In this case, preserve the Python code snippet evaluating to - # the current pith as is. + # In this case... else: + # Name of this local variable. + pith_curr_var_name = PITH_INDEX_TO_VAR_NAME[ + pith_curr_var_name_index] + + # Preserve the Python code snippet evaluating to the value + # of the current pith as is. pith_curr_assign_expr = pith_curr_expr # ............{ UNION }............ @@ -1262,7 +1285,8 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: hint_curr_expr=add_func_scope_types( types=hint_childs_nonpep, func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=( + EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL), ), )) @@ -1362,7 +1386,7 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: # Origin type of this sequence hint. cls=get_hint_pep_origin_type_isinstanceable(hint_curr), func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL, ) # print(f'Sequence type hint {hint_curr} origin type scoped: {hint_curr_expr}') @@ -1548,7 +1572,7 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: # Origin type of this sequence. cls=get_hint_pep_origin_type_isinstanceable(hint_curr), func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL, ) # 2-tuple of the possibly ignorable insane child key and @@ -1587,6 +1611,10 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: if hint_child_key: # If this child value hint is also unignorable... if hint_child_value: + # Increase the indentation level of code + # type-checking this child value pith. + indent_level_child += 1 + # Increment the integer suffixing the name of a # unique local variable storing the value of # this child key pith *BEFORE* defining this @@ -1611,10 +1639,6 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: # _enqueue_hint_child() closure. hint_child = hint_child_value - # Increase the indentation level of code - # type-checking this child value pith. - indent_level_child += 1 - # Placeholder string to be subsequently replaced # by code type-checking this child value pith # against this hint. @@ -1840,7 +1864,8 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: hint_curr_expr = add_func_scope_type_or_types( type_or_types=hint_child, # type: ignore[arg-type] func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=( + EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL), ) # Else, this superclass is *NOT* actually a class. By # process of elimination and the validation already @@ -1959,7 +1984,8 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: hint_curr_expr=add_func_scope_type( cls=hint_curr, func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=( + EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL), ), ) # print(f'{hint_curr_exception_prefix} PEP generic {repr(hint)} handled.') @@ -2005,7 +2031,8 @@ def _enqueue_hint_child(pith_child_expr: str) -> str: for hint_child in hint_childs }, func_scope=func_wrapper_scope, - exception_prefix=EXCEPTION_PREFIX_HINT, + exception_prefix=( + EXCEPTION_PREFIX_FUNC_WRAPPER_LOCAL), ), ) diff --git a/beartype/_check/code/snip/codesnipcls.py b/beartype/_check/code/snip/codesnipcls.py index b8af14be..2427c5aa 100644 --- a/beartype/_check/code/snip/codesnipcls.py +++ b/beartype/_check/code/snip/codesnipcls.py @@ -39,13 +39,6 @@ def __missing__(self, pith_index: int) -> str: and ``]``-delimited attempt to access a local pith variable name uniquely identified by the passed 1-based index. - Caveats - ------- - **This method intentionally prohibits 0.** Why? Because the existing - :data:`beartype._check.checkmagic.VAR_NAME_PITH_ROOT` string global - already efficiently caches the local root pith variable name, which is - sufficiently common to warrant globalization. - Parameters ---------- pith_index : int @@ -63,11 +56,10 @@ def __missing__(self, pith_index: int) -> str: If either: * ``pith_level`` is *not* an integer. - * ``pith_level`` is a **non-positive integer** (i.e., is less than - or equal to 0). + * ``pith_level`` is a **negative integer** (i.e., less than 0). ''' assert isinstance(pith_index, int), f'{repr(pith_index)} not integer.' - assert pith_index > 0, f'{pith_index} <= 0.' + assert pith_index >= 0, f'{pith_index} < 0.' # print(f'Generating indentation level {indent_level}...') # Prospective name of this local pith variable. diff --git a/beartype_test/a00_unit/a60_check/a00_code/snip/test_codesnipcls.py b/beartype_test/a00_unit/a60_check/a00_code/snip/test_codesnipcls.py index 2591ece7..4848c2f3 100644 --- a/beartype_test/a00_unit/a60_check/a00_code/snip/test_codesnipcls.py +++ b/beartype_test/a00_unit/a60_check/a00_code/snip/test_codesnipcls.py @@ -45,9 +45,7 @@ def test_pith_index_to_var_name() -> None: with raises(AssertionError): PITH_INDEX_TO_VAR_NAME[2.34] - # Assert that attempting to index this dictionary by non-positive indices + # Assert that attempting to index this dictionary by negative indices # raises the expected exception. - with raises(AssertionError): - PITH_INDEX_TO_VAR_NAME[0] with raises(AssertionError): PITH_INDEX_TO_VAR_NAME[-1] diff --git a/beartype_test/a00_unit/data/hint/pep/proposal/_data_pep585.py b/beartype_test/a00_unit/data/hint/pep/proposal/_data_pep585.py index 3c3f3fad..0bc7a452 100644 --- a/beartype_test/a00_unit/data/hint/pep/proposal/_data_pep585.py +++ b/beartype_test/a00_unit/data/hint/pep/proposal/_data_pep585.py @@ -758,7 +758,41 @@ def __len__(self) -> bool: ), ), - # Nested dictionaries of various kinds. + # Nested dictionaries of tuples. + HintPepMetadata( + hint=dict[tuple[int, float], str], + pep_sign=HintSignDict, + isinstanceable_type=dict, + is_pep585_builtin_subscripted=True, + piths_meta=( + # Dictionary mapping 2-tuples of integers and floating-point + # numbers to strings. + HintPithSatisfiedMetadata({ + (0xBEEFBABE, 42.42): ( + 'Obedient to the sweep of odorous winds'), + }), + # String constant. + HintPithUnsatisfiedMetadata( + 'Upon resplendent clouds, so rapidly'), + # Dictionary mapping 2-tuples of integers and floating-point + # numbers to byte strings. + HintPithUnsatisfiedMetadata( + pith={ + (0xBABEBEEF, 24.24): ( + b'Along the dark and ruffled waters fled'), + }, + # Match that the exception message raised for this object + # declares all key-value pairs on the path to the value + # violating this hint. + exception_str_match_regexes=( + r'\bkey tuple \(3133062895, 24.24\)', + r"\bvalue bytes b'Along the dark and ruffled waters fled'", + ), + ), + ), + ), + + # Nested dictionaries of nested dictionaries of... you get the idea. HintPepMetadata( hint=dict[int, Mapping[str, MutableMapping[bytes, bool]]], pep_sign=HintSignDict, diff --git a/beartype_test/a00_unit/data/hint/pep/proposal/data_pep484.py b/beartype_test/a00_unit/data/hint/pep/proposal/data_pep484.py index 436f1bc7..d54bdd57 100644 --- a/beartype_test/a00_unit/data/hint/pep/proposal/data_pep484.py +++ b/beartype_test/a00_unit/data/hint/pep/proposal/data_pep484.py @@ -1080,7 +1080,41 @@ def hints_pep484_meta() -> 'List[HintPepMetadata]': ), ), - # Nested dictionaries of various kinds. + # Nested dictionaries of tuples. + HintPepMetadata( + hint=Dict[Tuple[int, float], str], + pep_sign=HintSignDict, + warning_type=PEP585_DEPRECATION_WARNING, + isinstanceable_type=dict, + piths_meta=( + # Dictionary mapping 2-tuples of integers and floating-point + # numbers to strings. + HintPithSatisfiedMetadata({ + (0xBEEFBABE, 42.42): ( + 'Obedient to the sweep of odorous winds'), + }), + # String constant. + HintPithUnsatisfiedMetadata( + 'Upon resplendent clouds, so rapidly'), + # Dictionary mapping 2-tuples of integers and floating-point + # numbers to byte strings. + HintPithUnsatisfiedMetadata( + pith={ + (0xBABEBEEF, 24.24): ( + b'Along the dark and ruffled waters fled'), + }, + # Match that the exception message raised for this object + # declares all key-value pairs on the path to the value + # violating this hint. + exception_str_match_regexes=( + r'\bkey tuple \(3133062895, 24.24\)', + r"\bvalue bytes b'Along the dark and ruffled waters fled'", + ), + ), + ), + ), + + # Nested dictionaries of nested dictionaries of... you get the idea. HintPepMetadata( hint=Dict[int, Mapping[str, MutableMapping[bytes, bool]]], pep_sign=HintSignDict,