From 7876a87a2da8eb66504a34c36aa49f5c23148046 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:31:30 +0300 Subject: [PATCH 01/23] Type Annotations for RestrictedPython --- .meta.toml | 2 +- setup.py | 8 +- src/RestrictedPython/Utilities.py | 4 +- src/RestrictedPython/_compat.py | 2 + src/RestrictedPython/compile.py | 72 ++++---- src/RestrictedPython/transformer.py | 248 +++++++++++++++------------- tox.ini | 4 +- 7 files changed, 184 insertions(+), 156 deletions(-) diff --git a/.meta.toml b/.meta.toml index c5b68d7f..3ba9ecbd 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py311-datetime,py312,py313,py314,coverage", + "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index 45bf79d3..a7d2ec2b 100644 --- a/setup.py +++ b/setup.py @@ -39,17 +39,20 @@ def read(*rnames): 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', 'Programming Language :: Python :: Implementation :: CPython', 'Topic :: Security', + 'Typing :: Typed', ], keywords='restricted execution security untrusted code', author='Zope Foundation and Contributors', author_email='zope-dev@zope.dev', + maintainer='Zope Foundation, Plone Foundation and Contributors', + maintainer_email='security@plone.org', project_urls={ "Documentation": "https://restrictedpython.readthedocs.io/", "Source": "https://github.com/zopefoundation/RestrictedPython", @@ -59,9 +62,10 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.9, <3.15", + python_requires=">=3.10, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], + 'typecheck': ['mypy', 'typeshed'], 'docs': ['Sphinx', 'furo'], }, include_package_data=True, diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index 26d73d15..e07f8e49 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -14,6 +14,7 @@ import math import random import string +from collections.abc import Iterable utility_builtins = {} @@ -75,7 +76,8 @@ def test(*args): utility_builtins['test'] = test -def reorder(s, with_=None, without=()): +def reorder(s: Iterable, with_: Iterable | None = None, + without: Iterable | None = None) -> Iterable: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/_compat.py b/src/RestrictedPython/_compat.py index 2d85cc46..2c5641e2 100644 --- a/src/RestrictedPython/_compat.py +++ b/src/RestrictedPython/_compat.py @@ -6,5 +6,7 @@ IS_PY310_OR_GREATER = _version.major == 3 and _version.minor >= 10 IS_PY311_OR_GREATER = _version.major == 3 and _version.minor >= 11 IS_PY312_OR_GREATER = _version.major == 3 and _version.minor >= 12 +IS_PY313_OR_GREATER = _version.major == 3 and _version.minor >= 13 +IS_PY314_OR_GREATER = _version.major == 3 and _version.minor >= 14 IS_CPYTHON = platform.python_implementation() == 'CPython' diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 3253b8c9..0172023a 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,11 +1,19 @@ import ast import warnings from collections import namedtuple +from os import PathLike +from typing import Any +from typing import Literal +from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON from RestrictedPython.transformer import RestrictingNodeTransformer +# Temporary workaround for missing _typeshed +ReadableBuffer: TypeAlias = bytes | bytearray + + CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') syntax_error_template = ( @@ -18,12 +26,12 @@ def _compile_restricted_mode( - source, - filename='', - mode="exec", - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: Literal["exec", "eval", "single"] = "exec", + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: if not IS_CPYTHON: warnings.warn_explicit( @@ -78,11 +86,11 @@ def _compile_restricted_mode( def compile_restricted_exec( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( source, @@ -94,11 +102,11 @@ def compile_restricted_exec( def compile_restricted_eval( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( source, @@ -110,11 +118,11 @@ def compile_restricted_eval( def compile_restricted_single( - source, - filename='', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( source, @@ -128,12 +136,12 @@ def compile_restricted_single( def compile_restricted_function( p, # parameters body, - name, - filename='', + name: str, + filename: str | ReadableBuffer | PathLike[Any] = '', globalize=None, # List of globals (e.g. ['here', 'context', ...]) - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Compile a restricted code object for a function. Documentation see: @@ -181,12 +189,12 @@ def compile_restricted_function( def compile_restricted( - source, - filename='', - mode='exec', - flags=0, - dont_inherit=False, - policy=RestrictingNodeTransformer): + source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: str = 'exec', + flags: int = 0, + dont_inherit: bool = False, + policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 7fd30164..5ecc5f63 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -113,7 +113,7 @@ # When new ast nodes are generated they have no 'lineno', 'end_lineno', # 'col_offset' and 'end_col_offset'. This function copies these fields from the # incoming node: -def copy_locations(new_node, old_node): +def copy_locations(new_node: ast.AST, old_node: ast.AST) -> None: assert 'lineno' in new_node._attributes new_node.lineno = old_node.lineno @@ -151,7 +151,11 @@ def new_print_scope(self): class RestrictingNodeTransformer(ast.NodeTransformer): - def __init__(self, errors=None, warnings=None, used_names=None): + def __init__(self, + errors: list[str] | None = None, + warnings: list[str] | None = None, + used_names: dict[str, + str] | None = None): super().__init__() self.errors = [] if errors is None else errors self.warnings = [] if warnings is None else warnings @@ -168,26 +172,26 @@ def __init__(self, errors=None, warnings=None, used_names=None): self.print_info = PrintInfo() - def gen_tmp_name(self): + def gen_tmp_name(self) -> str: # 'check_name' ensures that no variable is prefixed with '_'. # => Its safe to use '_tmp..' as a temporary variable. name = '_tmp%i' % self._tmp_idx self._tmp_idx += 1 return name - def error(self, node, info): + def error(self, node: ast.AST, info: str) -> None: """Record a security error discovered during transformation.""" lineno = getattr(node, 'lineno', None) self.errors.append( f'Line {lineno}: {info}') - def warn(self, node, info): - """Record a security error discovered during transformation.""" + def warn(self, node: ast.AST, info: str) -> None: + """Record a security warning discovered during transformation.""" lineno = getattr(node, 'lineno', None) self.warnings.append( f'Line {lineno}: {info}') - def guard_iter(self, node): + def guard_iter(self, node: ast.AST) -> ast.AST: """ Converts: for x in expr @@ -218,10 +222,10 @@ def guard_iter(self, node): node.iter = new_iter return node - def is_starred(self, ob): + def is_starred(self, ob: ast.AST) -> bool: return isinstance(ob, ast.Starred) - def gen_unpack_spec(self, tpl): + def gen_unpack_spec(self, tpl: ast.Tuple) -> ast.Dict: """Generate a specification for 'guarded_unpack_sequence'. This spec is used to protect sequence unpacking. @@ -297,14 +301,18 @@ def gen_unpack_spec(self, tpl): return spec - def protect_unpack_sequence(self, target, value): + def protect_unpack_sequence( + self, + target: ast.Tuple, + value: ast.AST) -> ast.Call: spec = self.gen_unpack_spec(target) return ast.Call( func=ast.Name('_unpack_sequence_', ast.Load()), args=[value, spec, ast.Name('_getiter_', ast.Load())], keywords=[]) - def gen_unpack_wrapper(self, node, target): + def gen_unpack_wrapper(self, node: ast.AST, + target: ast.Tuple) -> tuple[ast.Name, ast.Try]: """Helper function to protect tuple unpacks. node: used to copy the locations for the new nodes. @@ -353,13 +361,13 @@ def gen_unpack_wrapper(self, node, target): return (tmp_target, cleanup) - def gen_none_node(self): + def gen_none_node(self) -> ast.NameConstant: return ast.NameConstant(value=None) - def gen_del_stmt(self, name_to_del): + def gen_del_stmt(self, name_to_del: str) -> ast.Delete: return ast.Delete(targets=[ast.Name(name_to_del, ast.Del())]) - def transform_slice(self, slice_): + def transform_slice(self, slice_: ast.AST) -> ast.AST: """Transform slices into function parameters. ast.Slice nodes are only allowed within a ast.Subscript node. @@ -398,7 +406,7 @@ def transform_slice(self, slice_): args=args, keywords=[]) - elif isinstance(slice_, ast.ExtSlice): + elif isinstance(slice_, ast.Tuple): dims = ast.Tuple([], ast.Load()) for item in slice_.dims: dims.elts.append(self.transform_slice(item)) @@ -408,7 +416,11 @@ def transform_slice(self, slice_): # Index, Slice and ExtSlice are only defined Slice types. raise NotImplementedError(f"Unknown slice type: {slice_}") - def check_name(self, node, name, allow_magic_methods=False): + def check_name( + self, + node: ast.AST, + name: str, + allow_magic_methods: bool = False) -> None: """Check names if they are allowed. If ``allow_magic_methods is True`` names in `ALLOWED_FUNC_NAMES` @@ -433,7 +445,7 @@ def check_name(self, node, name, allow_magic_methods=False): elif name in FORBIDDEN_FUNC_NAMES: self.error(node, f'"{name}" is a reserved name.') - def check_function_argument_names(self, node): + def check_function_argument_names(self, node: ast.FunctionDef) -> None: for arg in node.args.args: self.check_name(node, arg.arg) @@ -446,7 +458,7 @@ def check_function_argument_names(self, node): for arg in node.args.kwonlyargs: self.check_name(node, arg.arg) - def check_import_names(self, node): + def check_import_names(self, node: ast.ImportFrom | ast.Import) -> ast.AST: """Check the names being imported. This is a protection against rebinding dunder names like @@ -463,7 +475,7 @@ def check_import_names(self, node): return self.node_contents_visit(node) - def inject_print_collector(self, node, position=0): + def inject_print_collector(self, node: ast.AST, position: int = 0) -> None: print_used = self.print_info.print_used printed_used = self.print_info.printed_used @@ -496,7 +508,7 @@ def inject_print_collector(self, node, position=0): # Special Functions for an ast.NodeTransformer - def generic_visit(self, node): + def generic_visit(self, node: ast.AST) -> ast.AST: """Reject ast nodes which do not have a corresponding `visit_` method. This is needed to prevent new ast nodes from new Python versions to be @@ -511,18 +523,18 @@ def generic_visit(self, node): ) self.not_allowed(node) - def not_allowed(self, node): + def not_allowed(self, node: ast.AST) -> None: self.error( node, f'{node.__class__.__name__} statements are not allowed.') - def node_contents_visit(self, node): + def node_contents_visit(self, node: ast.AST) -> ast.AST: """Visit the contents of a node.""" return super().generic_visit(node) # ast for Literals - def visit_Constant(self, node): + def visit_Constant(self, node: ast.Constant) -> ast.Constant | None: """Allow constant literals with restriction for Ellipsis. Constant replaces Num, Str, Bytes, NameConstant and Ellipsis in @@ -539,37 +551,37 @@ def visit_Constant(self, node): return return self.node_contents_visit(node) - def visit_Interactive(self, node): + def visit_Interactive(self, node: ast.Interactive) -> ast.AST: """Allow single mode without restrictions.""" return self.node_contents_visit(node) - def visit_List(self, node): + def visit_List(self, node: ast.List) -> ast.AST: """Allow list literals without restrictions.""" return self.node_contents_visit(node) - def visit_Tuple(self, node): + def visit_Tuple(self, node: ast.Tuple) -> ast.AST: """Allow tuple literals without restrictions.""" return self.node_contents_visit(node) - def visit_Set(self, node): + def visit_Set(self, node: ast.Set) -> ast.AST: """Allow set literals without restrictions.""" return self.node_contents_visit(node) - def visit_Dict(self, node): + def visit_Dict(self, node: ast.Dict) -> ast.AST: """Allow dict literals without restrictions.""" return self.node_contents_visit(node) - def visit_FormattedValue(self, node): + def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) - def visit_JoinedStr(self, node): + def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" return self.node_contents_visit(node) # ast for Variables - def visit_Name(self, node): + def visit_Name(self, node: ast.Name) -> ast.Name | None: """Prevents access to protected names. Converts use of the name 'printed' to this expression: '_print()' @@ -603,25 +615,25 @@ def visit_Name(self, node): self.check_name(node, node.id) return node - def visit_Load(self, node): + def visit_Load(self, node: ast.Load) -> ast.Load | None: """ """ return self.node_contents_visit(node) - def visit_Store(self, node): + def visit_Store(self, node: ast.Store) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Del(self, node): + def visit_Del(self, node: ast.Del) -> ast.Del: """ """ return self.node_contents_visit(node) - def visit_Starred(self, node): + def visit_Starred(self, node: ast.Starred) -> ast.AST: """ """ @@ -629,18 +641,18 @@ def visit_Starred(self, node): # Expressions - def visit_Expression(self, node): + def visit_Expression(self, node: ast.Expression) -> ast.AST: """Allow Expression statements without restrictions. They are in the AST when using the `eval` compile mode. """ return self.node_contents_visit(node) - def visit_Expr(self, node): + def visit_Expr(self, node: ast.Expr) -> ast.AST: """Allow Expr statements (any expression) without restrictions.""" return self.node_contents_visit(node) - def visit_UnaryOp(self, node): + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: """ UnaryOp (Unary Operations) is the overall element for: * Not --> which should be allowed @@ -649,135 +661,135 @@ def visit_UnaryOp(self, node): """ return self.node_contents_visit(node) - def visit_UAdd(self, node): + def visit_UAdd(self, node: ast.UAdd) -> ast.AST: """Allow positive notation of variables. (e.g. +var)""" return self.node_contents_visit(node) - def visit_USub(self, node): + def visit_USub(self, node: ast.USub) -> ast.AST: """Allow negative notation of variables. (e.g. -var)""" return self.node_contents_visit(node) - def visit_Not(self, node): + def visit_Not(self, node: ast.Not) -> ast.AST: """Allow the `not` operator.""" return self.node_contents_visit(node) - def visit_Invert(self, node): + def visit_Invert(self, node: ast.Invert) -> ast.AST: """Allow `~` expressions.""" return self.node_contents_visit(node) - def visit_BinOp(self, node): + def visit_BinOp(self, node: ast.BinOp) -> ast.AST: """Allow binary operations.""" return self.node_contents_visit(node) - def visit_Add(self, node): + def visit_Add(self, node: ast.Add) -> ast.AST: """Allow `+` expressions.""" return self.node_contents_visit(node) - def visit_Sub(self, node): + def visit_Sub(self, node: ast.Sub) -> ast.AST: """Allow `-` expressions.""" return self.node_contents_visit(node) - def visit_Mult(self, node): + def visit_Mult(self, node: ast.Mult) -> ast.AST: """Allow `*` expressions.""" return self.node_contents_visit(node) - def visit_Div(self, node): + def visit_Div(self, node: ast.Div) -> ast.AST: """Allow `/` expressions.""" return self.node_contents_visit(node) - def visit_FloorDiv(self, node): + def visit_FloorDiv(self, node: ast.FloorDiv) -> ast.AST: """Allow `//` expressions.""" return self.node_contents_visit(node) - def visit_Mod(self, node): + def visit_Mod(self, node: ast.Mod) -> ast.AST: """Allow `%` expressions.""" return self.node_contents_visit(node) - def visit_Pow(self, node): + def visit_Pow(self, node: ast.Pow) -> ast.AST: """Allow `**` expressions.""" return self.node_contents_visit(node) - def visit_LShift(self, node): + def visit_LShift(self, node: ast.LShift) -> ast.AST: """Allow `<<` expressions.""" return self.node_contents_visit(node) - def visit_RShift(self, node): + def visit_RShift(self, node: ast.RShift) -> ast.AST: """Allow `>>` expressions.""" return self.node_contents_visit(node) - def visit_BitOr(self, node): + def visit_BitOr(self, node: ast.BitOr) -> ast.AST: """Allow `|` expressions.""" return self.node_contents_visit(node) - def visit_BitXor(self, node): + def visit_BitXor(self, node: ast.BitXor) -> ast.AST: """Allow `^` expressions.""" return self.node_contents_visit(node) - def visit_BitAnd(self, node): + def visit_BitAnd(self, node: ast.BitAnd) -> ast.AST: """Allow `&` expressions.""" return self.node_contents_visit(node) - def visit_MatMult(self, node): + def visit_MatMult(self, node: ast.MatMult) -> ast.AST: """Allow multiplication (`@`).""" return self.node_contents_visit(node) - def visit_BoolOp(self, node): + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: """Allow bool operator without restrictions.""" return self.node_contents_visit(node) - def visit_And(self, node): + def visit_And(self, node: ast.And) -> ast.AST: """Allow bool operator `and` without restrictions.""" return self.node_contents_visit(node) - def visit_Or(self, node): + def visit_Or(self, node: ast.Or) -> ast.AST: """Allow bool operator `or` without restrictions.""" return self.node_contents_visit(node) - def visit_Compare(self, node): + def visit_Compare(self, node: ast.Compare) -> ast.AST: """Allow comparison expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Eq(self, node): + def visit_Eq(self, node: ast.Eq) -> ast.AST: """Allow == expressions.""" return self.node_contents_visit(node) - def visit_NotEq(self, node): + def visit_NotEq(self, node: ast.NotEq) -> ast.AST: """Allow != expressions.""" return self.node_contents_visit(node) - def visit_Lt(self, node): + def visit_Lt(self, node: ast.Lt) -> ast.AST: """Allow < expressions.""" return self.node_contents_visit(node) - def visit_LtE(self, node): + def visit_LtE(self, node: ast.LtE) -> ast.AST: """Allow <= expressions.""" return self.node_contents_visit(node) - def visit_Gt(self, node): + def visit_Gt(self, node: ast.Gt) -> ast.AST: """Allow > expressions.""" return self.node_contents_visit(node) - def visit_GtE(self, node): + def visit_GtE(self, node: ast.GtE) -> ast.AST: """Allow >= expressions.""" return self.node_contents_visit(node) - def visit_Is(self, node): + def visit_Is(self, node: ast.Is) -> ast.AST: """Allow `is` expressions.""" return self.node_contents_visit(node) - def visit_IsNot(self, node): + def visit_IsNot(self, node: ast.IsNot) -> ast.AST: """Allow `is not` expressions.""" return self.node_contents_visit(node) - def visit_In(self, node): + def visit_In(self, node: ast.In) -> ast.AST: """Allow `in` expressions.""" return self.node_contents_visit(node) - def visit_NotIn(self, node): + def visit_NotIn(self, node: ast.NotIn) -> ast.AST: """Allow `not in` expressions.""" return self.node_contents_visit(node) - def visit_Call(self, node): + def visit_Call(self, node: ast.Call) -> ast.AST: """Checks calls with '*args' and '**kwargs'. Note: The following happens only if '*args' or '**kwargs' is used. @@ -819,17 +831,17 @@ def visit_Call(self, node): copy_locations(node.func, node.args[0]) return node - def visit_keyword(self, node): + def visit_keyword(self, node: ast.keyword) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_IfExp(self, node): + def visit_IfExp(self, node: ast.IfExp) -> ast.AST: """Allow `if` expressions without restrictions.""" return self.node_contents_visit(node) - def visit_Attribute(self, node): + def visit_Attribute(self, node: ast.Attribute) -> ast.AST: """Checks and mutates attribute access/assignment. 'a.b' becomes '_getattr_(a, "b")' @@ -885,7 +897,7 @@ def visit_Attribute(self, node): # Subscripting - def visit_Subscript(self, node): + def visit_Subscript(self, node: ast.Subscript) -> ast.AST: """Transforms all kinds of subscripts. 'foo[bar]' becomes '_getitem_(foo, bar)' @@ -930,19 +942,19 @@ def visit_Subscript(self, node): raise NotImplementedError( f"Unknown ctx type: {type(node.ctx)}") - def visit_Index(self, node): + def visit_Index(self, node: ast.Index) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Slice(self, node): + def visit_Slice(self, node: ast.Slice) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_ExtSlice(self, node): + def visit_ExtSlice(self, node: ast.ExtSlice) -> ast.AST: """ """ @@ -950,31 +962,31 @@ def visit_ExtSlice(self, node): # Comprehensions - def visit_ListComp(self, node): + def visit_ListComp(self, node: ast.ListComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_SetComp(self, node): + def visit_SetComp(self, node: ast.SetComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_GeneratorExp(self, node): + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_DictComp(self, node): + def visit_DictComp(self, node: ast.DictComp) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_comprehension(self, node): + def visit_comprehension(self, node: ast.comprehension) -> ast.AST: """ """ @@ -982,7 +994,7 @@ def visit_comprehension(self, node): # Statements - def visit_Assign(self, node): + def visit_Assign(self, node: ast.Assign) -> ast.AST: """ """ @@ -1031,7 +1043,7 @@ def visit_Assign(self, node): return new_nodes - def visit_AugAssign(self, node): + def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST: """Forbid certain kinds of AugAssign According to the language reference (and ast.c) the following nodes @@ -1082,75 +1094,75 @@ def visit_AugAssign(self, node): raise NotImplementedError( f"Unknown target type: {type(node.target)}") - def visit_Raise(self, node): + def visit_Raise(self, node: ast.Raise) -> ast.AST: """Allow `raise` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Assert(self, node): + def visit_Assert(self, node: ast.Assert) -> ast.AST: """Allow assert statements without restrictions.""" return self.node_contents_visit(node) - def visit_Delete(self, node): + def visit_Delete(self, node: ast.Delete) -> ast.AST: """Allow `del` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Pass(self, node): + def visit_Pass(self, node: ast.Pass) -> ast.AST: """Allow `pass` statements without restrictions.""" return self.node_contents_visit(node) # Imports - def visit_Import(self, node): + def visit_Import(self, node: ast.Import) -> ast.AST: """Allow `import` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_ImportFrom(self, node): + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST: """Allow `import from` statements with restrictions. See check_import_names.""" return self.check_import_names(node) - def visit_alias(self, node): + def visit_alias(self, node: ast.alias) -> ast.AST: """Allow `as` statements in import and import from statements.""" return self.node_contents_visit(node) # Control flow - def visit_If(self, node): + def visit_If(self, node: ast.If) -> ast.AST: """Allow `if` statements without restrictions.""" return self.node_contents_visit(node) - def visit_For(self, node): + def visit_For(self, node: ast.For) -> ast.AST: """Allow `for` statements with some restrictions.""" return self.guard_iter(node) - def visit_While(self, node): + def visit_While(self, node: ast.While) -> ast.AST: """Allow `while` statements.""" return self.node_contents_visit(node) - def visit_Break(self, node): + def visit_Break(self, node: ast.Break) -> ast.AST: """Allow `break` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Continue(self, node): + def visit_Continue(self, node: ast.Continue) -> ast.AST: """Allow `continue` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Try(self, node): + def visit_Try(self, node: ast.Try) -> ast.AST: """Allow `try` without restrictions.""" return self.node_contents_visit(node) - def visit_TryStar(self, node): + def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape.""" self.not_allowed(node) - def visit_ExceptHandler(self, node): + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST: """Protect exception handlers.""" node = self.node_contents_visit(node) self.check_name(node, node.name) return node - def visit_With(self, node): + def visit_With(self, node: ast.With) -> ast.AST: """Protect tuple unpacking on with statements.""" node = self.node_contents_visit(node) @@ -1165,13 +1177,13 @@ def visit_With(self, node): return node - def visit_withitem(self, node): + def visit_withitem(self, node: ast.withitem) -> ast.AST: """Allow `with` statements (context managers) without restrictions.""" return self.node_contents_visit(node) # Function and class definitions - def visit_FunctionDef(self, node): + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: """Allow function definitions (`def`) with some restrictions.""" self.check_name(node, node.name, allow_magic_methods=True) self.check_function_argument_names(node) @@ -1181,44 +1193,44 @@ def visit_FunctionDef(self, node): self.inject_print_collector(node) return node - def visit_Lambda(self, node): + def visit_Lambda(self, node: ast.Lambda) -> ast.AST: """Allow lambda with some restrictions.""" self.check_function_argument_names(node) return self.node_contents_visit(node) - def visit_arguments(self, node): + def visit_arguments(self, node: ast.arguments) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_arg(self, node): + def visit_arg(self, node: ast.arg) -> ast.AST: """ """ return self.node_contents_visit(node) - def visit_Return(self, node): + def visit_Return(self, node: ast.Return) -> ast.AST: """Allow `return` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Yield(self, node): + def visit_Yield(self, node: ast.Yield) -> ast.AST: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_YieldFrom(self, node): + def visit_YieldFrom(self, node: ast.YieldFrom) -> ast.AST: """Allow `yield`statements without restrictions.""" return self.node_contents_visit(node) - def visit_Global(self, node): + def visit_Global(self, node: ast.Global) -> ast.AST: """Allow `global` statements without restrictions.""" return self.node_contents_visit(node) - def visit_Nonlocal(self, node): + def visit_Nonlocal(self, node: ast.Nonlocal) -> ast.AST: """Deny `nonlocal` statements.""" self.not_allowed(node) - def visit_ClassDef(self, node): + def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST: """Check the name of a class definition.""" self.check_name(node, node.name) node = self.node_contents_visit(node) @@ -1235,7 +1247,7 @@ class {0.name}(metaclass=__metaclass__): new_class_node.decorator_list = node.decorator_list return new_class_node - def visit_Module(self, node): + def visit_Module(self, node: ast.Module) -> ast.AST: """Add the print_collector (only if print is used) at the top.""" node = self.node_contents_visit(node) @@ -1253,25 +1265,25 @@ def visit_Module(self, node): # Async und await - def visit_AsyncFunctionDef(self, node): + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST: """Deny async functions.""" self.not_allowed(node) - def visit_Await(self, node): + def visit_Await(self, node: ast.Await) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncFor(self, node): + def visit_AsyncFor(self, node: ast.AsyncFor) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) - def visit_AsyncWith(self, node): + def visit_AsyncWith(self, node: ast.AsyncWith) -> ast.AST: """Deny async functionality.""" self.not_allowed(node) # Assignment expressions (walrus operator ``:=``) # New in 3.8 - def visit_NamedExpr(self, node): + def visit_NamedExpr(self, node: ast.NamedExpr) -> ast.AST: """Allow assignment expressions under some circumstances.""" # while the grammar requires ``node.target`` to be a ``Name`` # the abstract syntax is more permissive and allows an ``expr``. diff --git a/tox.ini b/tox.ini index 4015b3a7..1f3fce15 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ envlist = py314 docs coverage - py311-datetime + py314-datetime combined-coverage [testenv] @@ -92,7 +92,7 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:docs] -basepython = python3 +basepython = python3.14 skip_install = false commands_pre = commands = From 1cd61981dcb7ef4730a9765141ace742690b49d5 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:36:48 +0300 Subject: [PATCH 02/23] isinstance check with ExtSlice and Tuple as for older Python Versions --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 5ecc5f63..990bdfa1 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -406,7 +406,7 @@ def transform_slice(self, slice_: ast.AST) -> ast.AST: args=args, keywords=[]) - elif isinstance(slice_, ast.Tuple): + elif isinstance(slice_, (ast.Tuple, ast.ExtSlice)): dims = ast.Tuple([], ast.Load()) for item in slice_.dims: dims.elts.append(self.transform_slice(item)) From e2599abe8144feb3aa4c5c77c7f5d041e939088d Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 11:41:47 +0300 Subject: [PATCH 03/23] liniting --- src/RestrictedPython/compile.py | 44 ++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 0172023a..83050a8e 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,5 +1,9 @@ import ast import warnings +from ast import Expression +from ast import Interactive +from ast import Module +from ast import NodeTransformer from collections import namedtuple from os import PathLike from typing import Any @@ -26,12 +30,13 @@ def _compile_restricted_mode( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', mode: Literal["exec", "eval", "single"] = "exec", flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: if not IS_CPYTHON: warnings.warn_explicit( @@ -47,13 +52,13 @@ def _compile_restricted_mode( dont_inherit=dont_inherit) elif issubclass(policy, RestrictingNodeTransformer): c_ast = None - allowed_source_types = [str, ast.Module] + allowed_source_types = [str, Module] if not issubclass(type(source), tuple(allowed_source_types)): raise TypeError('Not allowed source type: ' '"{0.__class__.__name__}".'.format(source)) c_ast = None # workaround for pypy issue https://bitbucket.org/pypy/pypy/issues/2552 - if isinstance(source, ast.Module): + if isinstance(source, Module): c_ast = source else: try: @@ -86,11 +91,12 @@ def _compile_restricted_mode( def compile_restricted_exec( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `exec`.""" return _compile_restricted_mode( source, @@ -102,11 +108,12 @@ def compile_restricted_exec( def compile_restricted_eval( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `eval`.""" return _compile_restricted_mode( source, @@ -118,11 +125,12 @@ def compile_restricted_eval( def compile_restricted_single( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, + source: str | ReadableBuffer | Module | Expression | Interactive, filename: str | ReadableBuffer | PathLike[Any] = '', flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile restricted for the mode `single`.""" return _compile_restricted_mode( source, @@ -141,7 +149,8 @@ def compile_restricted_function( globalize=None, # List of globals (e.g. ['here', 'context', ...]) flags: int = 0, dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + policy: ast.NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Compile a restricted code object for a function. Documentation see: @@ -189,12 +198,13 @@ def compile_restricted_function( def compile_restricted( - source: str | ReadableBuffer | ast.Module | ast.Expression | ast.Interactive, - filename: str | ReadableBuffer | PathLike[Any] = '', - mode: str = 'exec', - flags: int = 0, - dont_inherit: bool = False, - policy: ast.NodeTransformer = RestrictingNodeTransformer) -> CompileResult: + source: str | ReadableBuffer | Module | Expression | Interactive, + filename: str | ReadableBuffer | PathLike[Any] = '', + mode: str = 'exec', + flags: int = 0, + dont_inherit: bool = False, + policy: NodeTransformer = RestrictingNodeTransformer, +) -> CompileResult: """Replacement for the built-in compile() function. policy ... `ast.NodeTransformer` class defining the restrictions. From 4915d8e21705ea697c557daf45a7f17057be0606 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:05:09 +0300 Subject: [PATCH 04/23] Remove Python 3.9 as it end of life --- .github/workflows/tests.yml | 15 +++++++-------- .meta.toml | 2 +- tox.ini | 1 - 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8cb5ba8f..5abf4912 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,20 +24,19 @@ jobs: - ["windows", "windows-latest"] config: # [Python version, tox env] - - ["3.11", "release-check"] - - ["3.9", "py39"] + - ["3.14", "release-check"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] - ["3.13", "py313"] - ["3.14", "py314"] - - ["3.11", "docs"] - - ["3.11", "coverage"] - - ["3.11", "py311-datetime"] + - ["3.14", "docs"] + - ["3.14", "coverage"] + - ["3.14", "py314-datetime"] exclude: - - { os: ["windows", "windows-latest"], config: ["3.11", "release-check"] } - - { os: ["windows", "windows-latest"], config: ["3.11", "docs"] } - - { os: ["windows", "windows-latest"], config: ["3.11", "coverage"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "release-check"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "docs"] } + - { os: ["windows", "windows-latest"], config: ["3.14", "coverage"] } runs-on: ${{ matrix.os[1] }} if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name diff --git a/.meta.toml b/.meta.toml index 3ba9ecbd..f7000fbe 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/tox.ini b/tox.ini index 1f3fce15..71e73aa6 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ minversion = 3.18 envlist = release-check lint - py39 py310 py311 py312 From 3299a8bb301c16bfe36b9908bc0cbf6bf7b52b04 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:09:34 +0300 Subject: [PATCH 05/23] Remove License Cassifier, as they are deprecated --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a7d2ec2b..6e19db74 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ def read(*rnames): long_description_content_type='text/x-rst', classifiers=[ 'Development Status :: 6 - Mature', - 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', From 4d56e39735fdc56f850dc1195d3c8ad25b776a16 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:17 +0300 Subject: [PATCH 06/23] Add Comment for TryStar Annotation --- src/RestrictedPython/Utilities.py | 2 +- src/RestrictedPython/transformer.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/RestrictedPython/Utilities.py b/src/RestrictedPython/Utilities.py index e07f8e49..b966b4e2 100644 --- a/src/RestrictedPython/Utilities.py +++ b/src/RestrictedPython/Utilities.py @@ -77,7 +77,7 @@ def test(*args): def reorder(s: Iterable, with_: Iterable | None = None, - without: Iterable | None = None) -> Iterable: + without: Iterable = ()) -> Iterable: # s, with_, and without are sequences treated as sets. # The result is subtract(intersect(s, with_), without), # unless with_ is None, in which case it is subtract(s, without). diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 990bdfa1..abd9aa6f 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1153,7 +1153,10 @@ def visit_Try(self, node: ast.Try) -> ast.AST: return self.node_contents_visit(node) def visit_TryStar(self, node: ast.AST) -> ast.AST: - """Disallow `ExceptionGroup` due to a potential sandbox escape.""" + """Disallow `ExceptionGroup` due to a potential sandbox escape. + + TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. + """ self.not_allowed(node) def visit_ExceptHandler(self, node: ast.ExceptHandler) -> ast.AST: From 935a960b9a6e881f68e849c67364de0cda491008 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:41 +0300 Subject: [PATCH 07/23] Add Comment for TryStar Annotation --- src/RestrictedPython/transformer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index abd9aa6f..bcd87c5e 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1155,7 +1155,8 @@ def visit_Try(self, node: ast.Try) -> ast.AST: def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape. - TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. + TODO: Type Annotation for node when dropping support + for Python < 3.11 should be ast.TryStar. """ self.not_allowed(node) From e67da675036549242322289bc8844df535d8aff6 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:16:58 +0300 Subject: [PATCH 08/23] Add Comment for TryStar Annotation --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index bcd87c5e..126f1ff4 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -1155,7 +1155,7 @@ def visit_Try(self, node: ast.Try) -> ast.AST: def visit_TryStar(self, node: ast.AST) -> ast.AST: """Disallow `ExceptionGroup` due to a potential sandbox escape. - TODO: Type Annotation for node when dropping support + TODO: Type Annotation for node when dropping support for Python < 3.11 should be ast.TryStar. """ self.not_allowed(node) From 2a3d72829062af83148fe93a917ba9fb796a12eb Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:19:04 +0300 Subject: [PATCH 09/23] Add Changelog Entry --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 8823a783..041bc755 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Changes ---------------- - Nothing changed yet. +- Drop Support for Python 3.9 as it was EOL on 2025-10. +- Added basis support of Python 3.14 +- Added Type Annotations 8.1a1.dev0 (2025-03-20) From 337379507d03a7a5a707c211090a0a033abfd5f0 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 12:51:33 +0300 Subject: [PATCH 10/23] Base for Python 3.14 Updates --- docs/contributing/ast/python3_14.ast | 194 +++++++++++++++++++++ docs/contributing/changes_from313to314.rst | 5 + 2 files changed, 199 insertions(+) create mode 100644 docs/contributing/ast/python3_14.ast create mode 100644 docs/contributing/changes_from313to314.rst diff --git a/docs/contributing/ast/python3_14.ast b/docs/contributing/ast/python3_14.ast new file mode 100644 index 00000000..662f4f17 --- /dev/null +++ b/docs/contributing/ast/python3_14.ast @@ -0,0 +1,194 @@ +-- Python 3.14 AST +-- ASDL's 4 builtin types are: +-- identifier, int, string, constant + +module Python version "3.14" +{ + mod = Module(stmt* body, type_ignore* type_ignores) + | Interactive(stmt* body) + | Expression(expr body) + | FunctionType(expr* argtypes, expr returns) + + stmt = FunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + | AsyncFunctionDef(identifier name, + arguments args, + stmt* body, + expr* decorator_list, + expr? returns, + string? type_comment, + type_param* type_params) + + | ClassDef(identifier name, + expr* bases, + keyword* keywords, + stmt* body, + expr* decorator_list, + type_param* type_params) + | Return(expr? value) + + | Delete(expr* targets) + | Assign(expr* targets, expr value, string? type_comment) + | TypeAlias(expr name, type_param* type_params, expr value) + | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) + + -- use 'orelse' because else is a keyword in target languages + | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) + | While(expr test, stmt* body, stmt* orelse) + | If(expr test, stmt* body, stmt* orelse) + | With(withitem* items, stmt* body, string? type_comment) + | AsyncWith(withitem* items, stmt* body, string? type_comment) + + | Match(expr subject, match_case* cases) + + | Raise(expr? exc, expr? cause) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) + | Assert(expr test, expr? msg) + + | Import(alias* names) + | ImportFrom(identifier? module, alias* names, int? level) + + | Global(identifier* names) + | Nonlocal(identifier* names) + | Expr(expr value) + | Pass + | Break + | Continue + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- BoolOp() can use left & right? + expr = BoolOp(boolop op, expr* values) + | NamedExpr(expr target, expr value) + | BinOp(expr left, operator op, expr right) + | UnaryOp(unaryop op, expr operand) + | Lambda(arguments args, expr body) + | IfExp(expr test, expr body, expr orelse) + | Dict(expr* keys, expr* values) + | Set(expr* elts) + | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) + | DictComp(expr key, expr value, comprehension* generators) + | GeneratorExp(expr elt, comprehension* generators) + -- the grammar constrains where yield expressions can occur + | Await(expr value) + | Yield(expr? value) + | YieldFrom(expr value) + -- need sequences for compare to distinguish between + -- x < 4 < 3 and (x < 4) < 3 + | Compare(expr left, cmpop* ops, expr* comparators) + | Call(expr func, expr* args, keyword* keywords) + | FormattedValue(expr value, int conversion, expr? format_spec) + | JoinedStr(expr* values) + | Constant(constant value, string? kind) + + -- the following expression can appear in assignment context + | Attribute(expr value, identifier attr, expr_context ctx) + | Subscript(expr value, expr slice, expr_context ctx) + | Starred(expr value, expr_context ctx) + | Name(identifier id, expr_context ctx) + | List(expr* elts, expr_context ctx) + | Tuple(expr* elts, expr_context ctx) + + -- can appear only in Subscript + | Slice(expr? lower, expr? upper, expr? step) + + -- col_offset is the byte offset in the utf8 string the parser uses + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + expr_context = Load + | Store + | Del + + boolop = And + | Or + + operator = Add + | Sub + | Mult + | MatMult + | Div + | Mod + | Pow + | LShift + | RShift + | BitOr + | BitXor + | BitAnd + | FloorDiv + + unaryop = Invert + | Not + | UAdd + | USub + + cmpop = Eq + | NotEq + | Lt + | LtE + | Gt + | GtE + | Is + | IsNot + | In + | NotIn + + comprehension = (expr target, expr iter, expr* ifs, int is_async) + + excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + arguments = (arg* posonlyargs, + arg* args, + arg? vararg, + arg* kwonlyargs, + expr* kw_defaults, + arg? kwarg, + expr* defaults) + + arg = (identifier arg, expr? annotation, string? type_comment) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- keyword arguments supplied to call (NULL identifier for **kwargs) + keyword = (identifier? arg, expr value) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + -- import name with optional 'as' alias. + alias = (identifier name, identifier? asname) + attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) + + withitem = (expr context_expr, expr? optional_vars) + + match_case = (pattern pattern, expr? guard, stmt* body) + + pattern = MatchValue(expr value) + | MatchSingleton(constant value) + | MatchSequence(pattern* patterns) + | MatchMapping(expr* keys, pattern* patterns, identifier? rest) + | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns) + + | MatchStar(identifier? name) + -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys + + | MatchAs(pattern? pattern, identifier? name) + | MatchOr(pattern* patterns) + + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) + + type_ignore = TypeIgnore(int lineno, string tag) + + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) + attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) +} diff --git a/docs/contributing/changes_from313to314.rst b/docs/contributing/changes_from313to314.rst new file mode 100644 index 00000000..2690a880 --- /dev/null +++ b/docs/contributing/changes_from313to314.rst @@ -0,0 +1,5 @@ +Changes from Python 3.13 to Python 3.14 +--------------------------------------- + +.. literalinclude:: ast/python3_14.ast + :diff: ast/python3_13.ast From f86d8951aef75e19e6d8783b0c11d1c5117c7657 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 13:49:08 +0300 Subject: [PATCH 11/23] Update docs for Python 3.14 --- docs/contributing/ast/python3_14.ast | 4 +++- docs/contributing/index.rst | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/contributing/ast/python3_14.ast b/docs/contributing/ast/python3_14.ast index 662f4f17..2c1e0b78 100644 --- a/docs/contributing/ast/python3_14.ast +++ b/docs/contributing/ast/python3_14.ast @@ -74,7 +74,7 @@ module Python version "3.14" | UnaryOp(unaryop op, expr operand) | Lambda(arguments args, expr body) | IfExp(expr test, expr body, expr orelse) - | Dict(expr* keys, expr* values) + | Dict(expr?* keys, expr* values) | Set(expr* elts) | ListComp(expr elt, comprehension* generators) | SetComp(expr elt, comprehension* generators) @@ -89,7 +89,9 @@ module Python version "3.14" | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords) | FormattedValue(expr value, int conversion, expr? format_spec) + | Interpolation(expr value, constant str, int conversion, expr? format_spec) | JoinedStr(expr* values) + | TemplateStr(expr* values) | Constant(constant value, string? kind) -- the following expression can appear in assignment context diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst index 23e93b22..1a5cbdc9 100644 --- a/docs/contributing/index.rst +++ b/docs/contributing/index.rst @@ -103,6 +103,7 @@ A (modified style) Copy of all Abstract Grammar Definitions for the Python versi changes_from310to311 changes_from311to312 changes_from312to313 + changes_from313to314 .. _understand: @@ -235,11 +236,11 @@ Technical Backgrounds - Links to External Documentation * AST Grammar of Python (`Status of Python Versions`_) + * `Python 3.14 AST`_ (EOL 2030-10) * `Python 3.13 AST`_ (EOL 2029-10) * `Python 3.12 AST`_ (EOL 2028-10) * `Python 3.11 AST`_ (EOL 2027-10) * `Python 3.10 AST`_ (EOL 2026-10) - * `Python 3.9 AST`_ (EOL 2025-10) * `AST NodeVistiors Class`_ * `AST NodeTransformer Class`_ @@ -257,6 +258,8 @@ Todos .. _`What's new in Python`: https://docs.python.org/3/whatsnew/ +.. _`What's new in Python 3.14`: https://docs.python.org/3.14/whatsnew/3.14.html + .. _`What's new in Python 3.13`: https://docs.python.org/3.13/whatsnew/3.13.html .. _`What's new in Python 3.12`: https://docs.python.org/3.12/whatsnew/3.12.html @@ -281,6 +284,8 @@ Todos .. _`Python 3 AST`: https://docs.python.org/3/library/ast.html#abstract-grammar +.. _`Python 3.14 AST`: https://docs.python.org/3.14/library/ast.html#abstract-grammar + .. _`Python 3.13 AST`: https://docs.python.org/3.13/library/ast.html#abstract-grammar .. _`Python 3.12 AST`: https://docs.python.org/3.12/library/ast.html#abstract-grammar From b4ceb35e4c842ec06fefd2728986989e98728c2e Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 14:08:34 +0300 Subject: [PATCH 12/23] add provisional visit_TempalteStr and visit_Interpolation to transformer to start looking into it --- src/RestrictedPython/transformer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 126f1ff4..cffce5ac 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -575,6 +575,26 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: """Allow f-strings without restrictions.""" return self.node_contents_visit(node) + def visit_TemplateStr(self, node: ast.AST) -> ast.AST: + """Allow template strings without restrictions. + + TODO: Review security implications of template strings. + TODO: Change Type Annotation to ast.TemplateStr when + Support for Python 3.13 is dropped. + """ + return self.not_allowed(node) + # return self.node_contents_visit(node) + + def visit_InterpolatedStr(self, node: ast.AST) -> ast.AST: + """Allow interpolated strings without restrictions. + + TODO: Review security implications of interpolated strings. + TODO: Change Type Annotation to ast.InterpolatedStr when + Support for Python 3.13 is dropped. + """ + return self.not_allowed(node) + # return self.node_contents_visit(node) + def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" return self.node_contents_visit(node) From 43941ee0bb52b0225258403fe1d20b6951c2a56e Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 16:41:07 +0300 Subject: [PATCH 13/23] Disable t-strings --- src/RestrictedPython/transformer.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index cffce5ac..d88fbd9a 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,23 +576,30 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Allow template strings without restrictions. - - TODO: Review security implications of template strings. + """Template strings are not allowed by default. + Even so, that template strings can be useful in context of Template Engines + A Template String itself is not executed itself, but it contain expressions + and need additional template rendering logic applied to it to be useful. + Those rendering logic would be affected by RestrictedPython as well. + + TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ - return self.not_allowed(node) + self.warn(node, 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') + self.not_allowed(node) # return self.node_contents_visit(node) - def visit_InterpolatedStr(self, node: ast.AST) -> ast.AST: - """Allow interpolated strings without restrictions. - - TODO: Review security implications of interpolated strings. - TODO: Change Type Annotation to ast.InterpolatedStr when + def visit_Interpolation(self, node: ast.AST) -> ast.AST: + """Interpolations are not allowed by default. + As Interpolations are part of Template Strings, they will not be reached in + context of RestrictedPython as Template Strings are not allowed. + + TODO: Deeper review of security implications of interpolated strings. + TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ - return self.not_allowed(node) + self.not_allowed(node) # return self.node_contents_visit(node) def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: From f0e2a060f682822c86ae9976549d110ddc20e956 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 13:41:44 +0000 Subject: [PATCH 14/23] Apply pre-commit code formatting --- src/RestrictedPython/transformer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index d88fbd9a..0093921c 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,25 +576,27 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Template strings are not allowed by default. + """Template strings are not allowed by default. Even so, that template strings can be useful in context of Template Engines - A Template String itself is not executed itself, but it contain expressions + A Template String itself is not executed itself, but it contain expressions and need additional template rendering logic applied to it to be useful. - Those rendering logic would be affected by RestrictedPython as well. - + Those rendering logic would be affected by RestrictedPython as well. + TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ - self.warn(node, 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') + self.warn( + node, + 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') self.not_allowed(node) # return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be reached in + As Interpolations are part of Template Strings, they will not be reached in context of RestrictedPython as Template Strings are not allowed. - + TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. From 26a218cb93377855ff57a20eab4874fea68e0a45 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:08:29 +0300 Subject: [PATCH 15/23] reactivate Template-Strings --- src/RestrictedPython/transformer.py | 23 ++++--- tests/transformer/test_tstring.py | 103 ++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 tests/transformer/test_tstring.py diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 0093921c..d092167d 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -577,9 +577,10 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """Template strings are not allowed by default. - Even so, that template strings can be useful in context of Template Engines - A Template String itself is not executed itself, but it contain expressions - and need additional template rendering logic applied to it to be useful. + Even so, that template strings can be useful in context of Template + Engines. A Template String itself is not executed itself, but it + contain expressions and need additional template rendering logic + applied to it to be useful. Those rendering logic would be affected by RestrictedPython as well. TODO: Deeper review of security implications of template strings. @@ -588,21 +589,23 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: """ self.warn( node, - 'TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.') - self.not_allowed(node) - # return self.node_contents_visit(node) + 'TemplateStr statements are not yet allowed, ' + 'please use f-strings or a real template engine instead.') + # self.not_allowed(node) + return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be reached in - context of RestrictedPython as Template Strings are not allowed. + As Interpolations are part of Template Strings, they will not be + reached in the context of RestrictedPython as Template Strings + ‚‚are not allowed. TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ - self.not_allowed(node) - # return self.node_contents_visit(node) + # self.not_allowed(node) + return self.node_contents_visit(node) def visit_JoinedStr(self, node: ast.JoinedStr) -> ast.AST: """Allow joined string without restrictions.""" diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py new file mode 100644 index 00000000..c53a3df9 --- /dev/null +++ b/tests/transformer/test_tstring.py @@ -0,0 +1,103 @@ +from string.templatelib import Template + +import pytest + +from RestrictedPython import compile_restricted_exec +from RestrictedPython._compat import IS_PY314_OR_GREATER +from RestrictedPython.Eval import default_guarded_getattr +from RestrictedPython.Eval import default_guarded_getiter +from RestrictedPython.PrintCollector import PrintCollector + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_transform(): + """It compiles a function call successfully and returns the used name.""" + + result = compile_restricted_exec('a = t"{max([1, 2, 3])}"') + assert result.errors == () + assert result.warnings == [ + 'Line 1: TemplateStr statements are not yet allowed, please use f-strings or a real template engine instead.'] # NOQA: E501 + assert result.code is not None + loc = {} + exec(result.code, {}, loc) + template = loc['a'] + assert isinstance(template, Template) + assert template.values == (3, ) + assert result.used_names == {'max': True} + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_visit_invalid_variable_name(): + """Accessing private attributes is forbidden. + + This is just a smoke test to validate that restricted exec is used + in the run-time evaluation of t-strings. + """ + result = compile_restricted_exec('t"{__init__}"') + assert result.errors == ( + 'Line 1: "__init__" is an invalid variable name because it starts with "_"', # NOQA: E501 + ) + + +t_string_self_documenting_expressions_example = """ +from datetime import date +from string.templatelib import Template, Interpolation + +user = 'eric_idle' +member_since = date(1975, 7, 31) + +def render_template(template: Template) -> str: + result = '' + for part in template: + if isinstance(part, Interpolation): + if isinstance(part.value, str): + result += part.value.upper() + else: + result += str(part.value) + else: + result += part.lower() + return result + +print(render_template(t'The User {user} is a member since {member_since}')) +""" + + +@pytest.mark.skipif( + not IS_PY314_OR_GREATER, + reason="t-strings were added in Python 3.14.", +) +def test_t_string_self_documenting_expressions(): + """Checks if t-string self-documenting expressions is checked.""" + result = compile_restricted_exec( + t_string_self_documenting_expressions_example, + ) + # assert result.errors == ( + # 'Line 13: TemplateStr statements are not allowed.', + # ) + # assert result.warnings == [ + # 'Line 13: TemplateStr statements are not yet allowed, please use ' + # 'f-strings or a real template engine instead.', + # "Line None: Prints, but never reads 'printed' variable." + # ] + # assert result.code is None + assert result.errors == () + assert result.warnings == [ + 'Line 20: TemplateStr statements are not yet allowed, ' + 'please use f-strings or a real template engine instead.', + "Line None: Prints, but never reads 'printed' variable."] + assert result.code is not None + + glb = { + '_print_': PrintCollector, + '_getattr_': default_guarded_getattr, + '_getiter_': default_guarded_getiter, + '_inplacevar_': lambda x: x, + } + exec(result.code, glb) + assert glb['_print']() == "user='eric_idle' member_since=datetime.date(1975, 7, 31)\n" # NOQA: E501 From a4e189ed945988c62c77d9f52ac2de2c77644be8 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:38:13 +0300 Subject: [PATCH 16/23] Update Documentation for TemplateStr and Interploation --- src/RestrictedPython/transformer.py | 22 +++++++++++----------- tests/transformer/test_tstring.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index d092167d..78c67156 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -576,14 +576,13 @@ def visit_FormattedValue(self, node: ast.FormattedValue) -> ast.AST: return self.node_contents_visit(node) def visit_TemplateStr(self, node: ast.AST) -> ast.AST: - """Template strings are not allowed by default. - Even so, that template strings can be useful in context of Template - Engines. A Template String itself is not executed itself, but it - contain expressions and need additional template rendering logic - applied to it to be useful. + """Template strings are allowed by default. + + As Template strings are a very basic template mechanism, that needs + additional rendering logic to be useful, they are not blocked by + default. Those rendering logic would be affected by RestrictedPython as well. - TODO: Deeper review of security implications of template strings. TODO: Change Type Annotation to ast.TemplateStr when Support for Python 3.13 is dropped. """ @@ -595,12 +594,13 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: return self.node_contents_visit(node) def visit_Interpolation(self, node: ast.AST) -> ast.AST: - """Interpolations are not allowed by default. - As Interpolations are part of Template Strings, they will not be - reached in the context of RestrictedPython as Template Strings - ‚‚are not allowed. + """Interpolations are allowed by default. + As Interpolations are part of Template Strings, they are needed + to be reached in the context of RestrictedPython as Template Strings + are allowed. As a user has to provide additional rendering logic + to make use of Template Strings, the security implications of + Interpolations are limited in the context of RestrictedPython. - TODO: Deeper review of security implications of interpolated strings. TODO: Change Type Annotation to ast.Interpolation when Support for Python 3.13 is dropped. """ diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index c53a3df9..7aa187e5 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -97,7 +97,7 @@ def test_t_string_self_documenting_expressions(): '_print_': PrintCollector, '_getattr_': default_guarded_getattr, '_getiter_': default_guarded_getiter, - '_inplacevar_': lambda x: x, + '_inplacevar_': lambda x, y, z: y + z, } exec(result.code, glb) - assert glb['_print']() == "user='eric_idle' member_since=datetime.date(1975, 7, 31)\n" # NOQA: E501 + assert glb['_print']() == "the user ERIC_IDLE is a member since 1975-07-31\n" # NOQA: E501 From fd0328a5cfaef06f6c93c273c0e62b7735719c42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 14:38:56 +0000 Subject: [PATCH 17/23] Apply pre-commit code formatting --- src/RestrictedPython/transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RestrictedPython/transformer.py b/src/RestrictedPython/transformer.py index 78c67156..2826b3ac 100644 --- a/src/RestrictedPython/transformer.py +++ b/src/RestrictedPython/transformer.py @@ -595,7 +595,7 @@ def visit_TemplateStr(self, node: ast.AST) -> ast.AST: def visit_Interpolation(self, node: ast.AST) -> ast.AST: """Interpolations are allowed by default. - As Interpolations are part of Template Strings, they are needed + As Interpolations are part of Template Strings, they are needed to be reached in the context of RestrictedPython as Template Strings are allowed. As a user has to provide additional rendering logic to make use of Template Strings, the security implications of From aceca07c2fd8246aec37f0ced75351905353e201 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:41:49 +0300 Subject: [PATCH 18/23] conditional import --- tests/transformer/test_tstring.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index 7aa187e5..de0ed356 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -1,5 +1,3 @@ -from string.templatelib import Template - import pytest from RestrictedPython import compile_restricted_exec @@ -9,6 +7,10 @@ from RestrictedPython.PrintCollector import PrintCollector +if IS_PY314_OR_GREATER: + from string.templatelib import Template + + @pytest.mark.skipif( not IS_PY314_OR_GREATER, reason="t-strings were added in Python 3.14.", From 3533c0f9d09601d1d8cb2a6cc5f9c58e3d11fd74 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 17:45:08 +0300 Subject: [PATCH 19/23] fix coverage numbers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 89faa35f..5fa3c8e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ branch = true source = ["RestrictedPython"] [tool.coverage.report] -fail_under = 97.3 +fail_under = 97.0 precision = 2 ignore_errors = true show_missing = true From e7503315ad2050c6cf94fdfbf96934020087697d Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Sat, 18 Oct 2025 18:17:35 +0300 Subject: [PATCH 20/23] readd Python 3.9 support --- .github/workflows/tests.yml | 1 + .meta.toml | 2 +- setup.py | 2 +- src/RestrictedPython/compile.py | 13 ++++++++++--- tox.ini | 1 + 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5abf4912..dd13cef3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,6 +25,7 @@ jobs: config: # [Python version, tox env] - ["3.14", "release-check"] + - ["3.9", "py39"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] diff --git a/.meta.toml b/.meta.toml index f7000fbe..3ba9ecbd 100644 --- a/.meta.toml +++ b/.meta.toml @@ -47,7 +47,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index 6e19db74..c01597e1 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.10, <3.15", + python_requires=">=3.9, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], 'typecheck': ['mypy', 'typeshed'], diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 83050a8e..07fd180a 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import ast import warnings from ast import Expression @@ -8,15 +10,20 @@ from os import PathLike from typing import Any from typing import Literal -from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON +from RestrictedPython._compat import IS_PY310_OR_GREATER from RestrictedPython.transformer import RestrictingNodeTransformer -# Temporary workaround for missing _typeshed -ReadableBuffer: TypeAlias = bytes | bytearray +if IS_PY310_OR_GREATER: + from typing import TypeAlias + # Temporary workaround for missing _typeshed + ReadableBuffer: TypeAlias = bytes | bytearray +else: + from typing_extensions import TypeAlias # type: ignore[import] + ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') diff --git a/tox.ini b/tox.ini index 71e73aa6..1f3fce15 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ minversion = 3.18 envlist = release-check lint + py39 py310 py311 py312 From 5b8285cdf901c6912b4082db9aa546f75f29a5c0 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Sun, 19 Oct 2025 16:29:19 +0200 Subject: [PATCH 21/23] - updating package files with zope/meta and fixing tests --- .github/workflows/tests.yml | 15 +++++++-------- .meta.toml | 3 ++- .pre-commit-config.yaml | 2 +- pyproject.toml | 2 +- setup.py | 2 +- src/RestrictedPython/compile.py | 2 +- tox.ini | 7 +++---- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3717ce65..3dc22730 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,20 +24,19 @@ jobs: - ["windows", "windows-latest"] config: # [Python version, tox env] - - ["3.14", "release-check"] - - ["3.9", "py39"] + - ["3.11", "release-check"] - ["3.10", "py310"] - ["3.11", "py311"] - ["3.12", "py312"] - ["3.13", "py313"] - ["3.14", "py314"] - - ["3.14", "docs"] - - ["3.14", "coverage"] - - ["3.14", "py314-datetime"] + - ["3.11", "docs"] + - ["3.11", "coverage"] + - ["3.11", "py311-datetime"] exclude: - - { os: ["windows", "windows-latest"], config: ["3.14", "release-check"] } - - { os: ["windows", "windows-latest"], config: ["3.14", "docs"] } - - { os: ["windows", "windows-latest"], config: ["3.14", "coverage"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "release-check"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "docs"] } + - { os: ["windows", "windows-latest"], config: ["3.11", "coverage"] } runs-on: ${{ matrix.os[1] }} if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name diff --git a/.meta.toml b/.meta.toml index e1f29d4d..35b7ea04 100644 --- a/.meta.toml +++ b/.meta.toml @@ -2,7 +2,7 @@ # https://github.com/zopefoundation/meta/tree/master/config/pure-python [meta] template = "pure-python" -commit-id = "72252845" +commit-id = "9d049229" [python] with-pypy = false @@ -11,6 +11,7 @@ with-sphinx-doctests = true with-windows = true with-future-python = true with-macos = false +oldest-python = "3.10" [tox] use-flake8 = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cbb2541a..5f86332c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: rev: v3.21.0 hooks: - id: pyupgrade - args: [--py39-plus] + args: [--py310-plus] - repo: https://github.com/isidentical/teyit rev: 0.4.3 hooks: diff --git a/pyproject.toml b/pyproject.toml index 5fa3c8e2..7b94ef64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ branch = true source = ["RestrictedPython"] [tool.coverage.report] -fail_under = 97.0 +fail_under = 97.2 precision = 2 ignore_errors = true show_missing = true diff --git a/setup.py b/setup.py index 5a3f97ff..dddd6d9d 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ def read(*rnames): packages=find_packages('src'), package_dir={'': 'src'}, install_requires=[], - python_requires=">=3.9, <3.15", + python_requires=">=3.10, <3.15", extras_require={ 'test': ['pytest', 'pytest-mock'], 'typecheck': ['mypy', 'typeshed'], diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 07fd180a..56f3553c 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -22,7 +22,7 @@ # Temporary workaround for missing _typeshed ReadableBuffer: TypeAlias = bytes | bytearray else: - from typing_extensions import TypeAlias # type: ignore[import] + from typing import TypeAlias # type: ignore[import] ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] CompileResult = namedtuple( diff --git a/tox.ini b/tox.ini index 1f3fce15..990f2601 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ minversion = 3.18 envlist = release-check lint - py39 py310 py311 py312 @@ -13,7 +12,7 @@ envlist = py314 docs coverage - py314-datetime + py311-datetime combined-coverage [testenv] @@ -52,7 +51,7 @@ commands = coverage combine coverage html coverage report -m --fail-under=100 -depends = py39,py310,py311,py311-datetime,py312,py313,py314,coverage +depends = py310,py311,py311-datetime,py312,py313,py314,coverage [testenv:setuptools-latest] basepython = python3 @@ -92,7 +91,7 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:docs] -basepython = python3.14 +basepython = python3 skip_install = false commands_pre = commands = From ac53335291d03319dee9ba9786d67a28aebb2f65 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Sun, 19 Oct 2025 16:34:12 +0200 Subject: [PATCH 22/23] - fix last test --- .meta.toml | 2 +- setup.py | 1 + src/RestrictedPython/compile.py | 12 +++--------- tests/transformer/test_tstring.py | 1 - 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.meta.toml b/.meta.toml index 35b7ea04..f9192772 100644 --- a/.meta.toml +++ b/.meta.toml @@ -48,7 +48,7 @@ testenv-additional = [ " coverage combine", " coverage html", " coverage report -m --fail-under=100", - "depends = py39,py310,py311,py312,py313,py314,py314-datetime,coverage", + "depends = py310,py311,py311-datetime,py312,py313,py314,coverage", ] coverage-command = "pytest --cov=src --cov=tests --cov-report= tests {posargs}" coverage-setenv = [ diff --git a/setup.py b/setup.py index dddd6d9d..b8b6f650 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ def read(*rnames): long_description_content_type='text/x-rst', classifiers=[ 'Development Status :: 6 - Mature', + 'License :: OSI Approved :: Zope Public License', 'Programming Language :: Python', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', diff --git a/src/RestrictedPython/compile.py b/src/RestrictedPython/compile.py index 56f3553c..8d29f3a9 100644 --- a/src/RestrictedPython/compile.py +++ b/src/RestrictedPython/compile.py @@ -10,20 +10,14 @@ from os import PathLike from typing import Any from typing import Literal +from typing import TypeAlias from RestrictedPython._compat import IS_CPYTHON -from RestrictedPython._compat import IS_PY310_OR_GREATER from RestrictedPython.transformer import RestrictingNodeTransformer -if IS_PY310_OR_GREATER: - from typing import TypeAlias - - # Temporary workaround for missing _typeshed - ReadableBuffer: TypeAlias = bytes | bytearray -else: - from typing import TypeAlias # type: ignore[import] - ReadableBuffer: TypeAlias = bytes | bytearray # type: ignore[no-redef] +# Temporary workaround for missing _typeshed +ReadableBuffer: TypeAlias = bytes | bytearray CompileResult = namedtuple( 'CompileResult', 'code, errors, warnings, used_names') diff --git a/tests/transformer/test_tstring.py b/tests/transformer/test_tstring.py index 87cfd968..3a2bf2d5 100644 --- a/tests/transformer/test_tstring.py +++ b/tests/transformer/test_tstring.py @@ -79,7 +79,6 @@ def test_t_string_self_documenting_expressions(): t_string_self_documenting_expressions_example, ) assert result.errors == () - assert result.warnings == [] assert result.code is not None glb = { From 0e1408cfe9ec30396c56297cd381ed41b424c9a6 Mon Sep 17 00:00:00 2001 From: Jens Vagelpohl Date: Mon, 20 Oct 2025 16:27:38 +0200 Subject: [PATCH 23/23] - expand change log entry to be more clear. --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0f72aed6..9a50d664 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,8 @@ Changes 8.2 (unreleased) ---------------- -- Added Type Annotations +- Add type annotations to the package code. + For clarification, restricted Python code does not support type annotations. 8.1 (2025-10-19)