Skip to content

gh-133403: Check Tools/build/deepfreeze.py with mypy #133802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 15, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/mypy.yml
Original file line number Diff line number Diff line change
@@ -14,10 +14,12 @@ on:
- "Lib/tomllib/**"
- "Misc/mypy/**"
- "Tools/build/compute-changes.py"
- "Tools/build/deepfreeze.py"
- "Tools/build/generate_sbom.py"
- "Tools/build/generate-build-details.py"
- "Tools/build/verify_ensurepip_wheels.py"
- "Tools/build/update_file.py"
- "Tools/build/umarshal.py"
- "Tools/cases_generator/**"
- "Tools/clinic/**"
- "Tools/jit/**"
2 changes: 1 addition & 1 deletion Tools/build/.ruff.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
extend = "../../.ruff.toml" # Inherit the project-wide settings

[per-file-target-version]
"deepfreeze.py" = "py310"
"deepfreeze.py" = "py311" # requires `code.co_exceptiontable`
"stable_abi.py" = "py311" # requires 'tomllib'

[format]
51 changes: 33 additions & 18 deletions Tools/build/deepfreeze.py
Original file line number Diff line number Diff line change
@@ -2,9 +2,12 @@

The script may be executed by _bootstrap_python interpreter.
Shared library extension modules are not available in that case.
On Windows, and in cross-compilation cases, it is executed
by Python 3.10, and 3.11 features are not available.
Requires 3.11+ to be executed,
because relies on `code.co_qualname` and `code.co_exceptiontable`.
"""

from __future__ import annotations

import argparse
import builtins
import collections
@@ -13,10 +16,14 @@
import re
import time
import types
from typing import TextIO

import umarshal

TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterator
from typing import Any, TextIO

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))

verbose = False
@@ -45,8 +52,8 @@ def make_string_literal(b: bytes) -> str:

next_code_version = 1

def get_localsplus(code: types.CodeType):
a = collections.defaultdict(int)
def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]:
a: collections.defaultdict[str, int] = collections.defaultdict(int)
for name in code.co_varnames:
a[name] |= CO_FAST_LOCAL
for name in code.co_cellvars:
@@ -136,7 +143,7 @@ def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]:
return identifiers, strings

@contextlib.contextmanager
def indent(self) -> None:
def indent(self) -> Iterator[None]:
save_level = self.level
try:
self.level += 1
@@ -148,7 +155,7 @@ def write(self, arg: str) -> None:
self.file.writelines((" "*self.level, arg, "\n"))

@contextlib.contextmanager
def block(self, prefix: str, suffix: str = "") -> None:
def block(self, prefix: str, suffix: str = "") -> Iterator[None]:
self.write(prefix + " {")
with self.indent():
yield
@@ -250,9 +257,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
co_names = self.generate(name + "_names", code.co_names)
co_filename = self.generate(name + "_filename", code.co_filename)
co_name = self.generate(name + "_name", code.co_name)
co_qualname = self.generate(name + "_qualname", code.co_qualname)
co_linetable = self.generate(name + "_linetable", code.co_linetable)
co_exceptiontable = self.generate(name + "_exceptiontable", code.co_exceptiontable)
# We use 3.10 for type checking, but this module requires 3.11
# TODO: bump python version for this script.
co_qualname = self.generate(
name + "_qualname",
code.co_qualname, # type: ignore[attr-defined]
)
co_exceptiontable = self.generate(
name + "_exceptiontable",
code.co_exceptiontable, # type: ignore[attr-defined]
)
# These fields are not directly accessible
localsplusnames, localspluskinds = get_localsplus(code)
co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames)
@@ -379,13 +394,13 @@ def generate_complex(self, name: str, z: complex) -> str:
self.write(f".cval = {{ {z.real}, {z.imag} }},")
return f"&{name}.ob_base"

def generate_frozenset(self, name: str, fs: frozenset[object]) -> str:
def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str:
try:
fs = sorted(fs)
fs_sorted = sorted(fs)
except TypeError:
# frozen set with incompatible types, fallback to repr()
fs = sorted(fs, key=repr)
ret = self.generate_tuple(name, tuple(fs))
fs_sorted = sorted(fs, key=repr)
ret = self.generate_tuple(name, tuple(fs_sorted))
self.write("// TODO: The above tuple should be a frozenset")
return ret

@@ -402,7 +417,7 @@ def generate(self, name: str, obj: object) -> str:
# print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}")
return self.cache[key]
self.misses += 1
if isinstance(obj, (types.CodeType, umarshal.Code)) :
if isinstance(obj, types.CodeType) :
Copy link
Member Author

@sobolevn sobolevn May 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here umarshal.Code won't work, because generate_code requires a lot more attrs than umarshal.Code has

val = self.generate_code(name, obj)
elif isinstance(obj, tuple):
val = self.generate_tuple(name, obj)
@@ -458,7 +473,7 @@ def decode_frozen_data(source: str) -> types.CodeType:
if re.match(FROZEN_DATA_LINE, line):
values.extend([int(x) for x in line.split(",") if x.strip()])
data = bytes(values)
return umarshal.loads(data)
return umarshal.loads(data) # type: ignore[no-any-return]


def generate(args: list[str], output: TextIO) -> None:
@@ -494,12 +509,12 @@ def generate(args: list[str], output: TextIO) -> None:
help="Input file and module name (required) in file:modname format")

@contextlib.contextmanager
def report_time(label: str):
t0 = time.time()
def report_time(label: str) -> Iterator[None]:
t0 = time.perf_counter()
try:
yield
finally:
t1 = time.time()
t1 = time.perf_counter()
if verbose:
print(f"{label}: {t1-t0:.3f} sec")

4 changes: 3 additions & 1 deletion Tools/build/mypy.ini
Original file line number Diff line number Diff line change
@@ -4,10 +4,12 @@
# .github/workflows/mypy.yml
files =
Tools/build/compute-changes.py,
Tools/build/deepfreeze.py,
Tools/build/generate-build-details.py,
Tools/build/generate_sbom.py,
Tools/build/verify_ensurepip_wheels.py,
Tools/build/update_file.py
Tools/build/update_file.py,
Tools/build/umarshal.py

pretty = True

11 changes: 6 additions & 5 deletions Tools/build/umarshal.py
Original file line number Diff line number Diff line change
@@ -145,12 +145,12 @@ def r_PyLong(self) -> int:
def r_float_bin(self) -> float:
buf = self.r_string(8)
import struct # Lazy import to avoid breaking UNIX build
return struct.unpack("d", buf)[0]
return struct.unpack("d", buf)[0] # type: ignore[no-any-return]

def r_float_str(self) -> float:
n = self.r_byte()
buf = self.r_string(n)
return ast.literal_eval(buf.decode("ascii"))
return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return]

def r_ref_reserve(self, flag: int) -> int:
if flag:
@@ -306,16 +306,17 @@ def loads(data: bytes) -> Any:
return r.r_object()


def main():
def main() -> None:
# Test
import marshal
import pprint
sample = {'foo': {(42, "bar", 3.14)}}
data = marshal.dumps(sample)
retval = loads(data)
assert retval == sample, retval
sample = main.__code__
data = marshal.dumps(sample)

sample2 = main.__code__
data = marshal.dumps(sample2)
retval = loads(data)
assert isinstance(retval, Code), retval
pprint.pprint(retval.__dict__)
Loading
Oops, something went wrong.