From def666278645bdbad6c79577e3a82e6d3038a5af Mon Sep 17 00:00:00 2001 From: carloea2 Date: Tue, 1 Sep 2026 17:09:26 -0600 Subject: [PATCH 1/3] feat(py2udf): add source forest --- .github/labeler.yml | 6 +- .github/workflows/build.yml | 8 + py2udf/.gitignore | 3 + py2udf/LICENSE | 205 ++ py2udf/NOTICE | 5 + py2udf/pyproject.toml | 58 + .../python/python_to_workflow/__init__.py | 18 + .../python_to_workflow/mosaic/__init__.py | 18 + .../python_to_workflow/mosaic/forest.py | 2551 +++++++++++++++++ .../python_to_workflow/mosaic/source.py | 162 ++ .../python_to_workflow/mosaic/test_forest.py | 259 ++ 11 files changed, 3290 insertions(+), 3 deletions(-) create mode 100644 py2udf/.gitignore create mode 100644 py2udf/LICENSE create mode 100644 py2udf/NOTICE create mode 100644 py2udf/pyproject.toml create mode 100644 py2udf/src/main/python/python_to_workflow/__init__.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/__init__.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/forest.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/source.py create mode 100644 py2udf/src/test/python/python_to_workflow/mosaic/test_forest.py diff --git a/.github/labeler.yml b/.github/labeler.yml index 65aa19f99ed..775a40bdb28 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -97,13 +97,14 @@ pyamber: # bumped requirements.txt would only get `dependencies` (no stack # mapping) and silently skip CI for the very deps it's changing. # - # Globs are scoped to amber/ so bin/ tooling Python (its own `infra` - # job) isn't dragged into the pyamber stack. + # The Python-to-workflow compiler shares this Python test stack with + # PyAmber. bin/ tooling Python retains its separate `infra` job. - changed-files: - any-glob-to-any-file: - 'amber/**/*.py' - 'amber/pyproject.toml' - 'amber/**/*requirements*.txt' + - 'py2udf/**' docs: - changed-files: @@ -150,4 +151,3 @@ fix: refactor: - head-branch: '^refactor' - diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0107997b39..34eb25b3e44 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1126,6 +1126,11 @@ jobs: # dev-dependency install above. run: | cd amber && ruff check src/main/python src/test/python && ruff format --check src/main/python src/test/python + - name: Lint the Python-to-workflow compiler + if: matrix.python-version == '3.12' + run: | + ruff check py2udf/src/main/python py2udf/src/test/python + ruff format --check py2udf/src/main/python py2udf/src/test/python - name: Install protoc # Version pinned in bin/protoc-version.txt. run: | @@ -1154,6 +1159,9 @@ jobs: LOGURU_LEVEL: ${{ runner.debug == '1' && 'DEBUG' || 'WARNING' }} run: | cd amber && pytest -m "not integration" --cov=src/main/python --cov-report=xml --junit-xml=junit.xml -sv + - name: Test the Python-to-workflow compiler + if: matrix.python-version == '3.12' + run: python -m pytest -q py2udf/src/test/python - name: Upload pyamber coverage to Codecov if: matrix.python-version == '3.12' && always() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 diff --git a/py2udf/.gitignore b/py2udf/.gitignore new file mode 100644 index 00000000000..7804abde72a --- /dev/null +++ b/py2udf/.gitignore @@ -0,0 +1,3 @@ +/build/ +/.ruff_cache/ +/src/main/python/*.egg-info/ diff --git a/py2udf/LICENSE b/py2udf/LICENSE new file mode 100644 index 00000000000..51f1829e5da --- /dev/null +++ b/py2udf/LICENSE @@ -0,0 +1,205 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- +THIRD-PARTY DEPENDENCIES +-------------------------------------------------------------------------------- diff --git a/py2udf/NOTICE b/py2udf/NOTICE new file mode 100644 index 00000000000..67ed88502c0 --- /dev/null +++ b/py2udf/NOTICE @@ -0,0 +1,5 @@ +Apache Texera (Incubating) +Copyright 2025-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/py2udf/pyproject.toml b/py2udf/pyproject.toml new file mode 100644 index 00000000000..845a99534a5 --- /dev/null +++ b/py2udf/pyproject.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "texera-python-to-workflow" +version = "0.1.0" +description = "Statement-level Python-to-Texera workflow compiler" +requires-python = ">=3.12" +license = "Apache-2.0" +license-files = ["LICENSE", "NOTICE"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3.12", +] + +[project.urls] +Homepage = "https://texera.io/" +Repository = "https://github.com/apache/texera" + +[tool.setuptools] +package-dir = {"" = "src/main/python"} + +[tool.setuptools.packages.find] +where = ["src/main/python"] +include = ["python_to_workflow", "python_to_workflow.mosaic*"] + +[tool.pytest.ini_options] +pythonpath = ["src/main/python"] +testpaths = ["src/test/python"] +addopts = "--import-mode=importlib" + +[tool.ruff] +line-length = 88 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "C90"] + +[tool.ruff.lint.mccabe] +max-complexity = 10 diff --git a/py2udf/src/main/python/python_to_workflow/__init__.py b/py2udf/src/main/python/python_to_workflow/__init__.py new file mode 100644 index 00000000000..e9b5ee84d1d --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Translate Python source into Texera workflows.""" diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/__init__.py b/py2udf/src/main/python/python_to_workflow/mosaic/__init__.py new file mode 100644 index 00000000000..46190f8404e --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Python-to-workflow compiler components.""" diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/forest.py b/py2udf/src/main/python/python_to_workflow/mosaic/forest.py new file mode 100644 index 00000000000..babc9b49dfb --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/forest.py @@ -0,0 +1,2551 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Source-exact Action/Parameter forests for Python statements. + +The public graph is deliberately small: executable ``Command`` and +``Expression`` actions alternate with non-executable ``Parameter`` source +containers. Python's AST is used only to discover that structure; AST helper +nodes, operators, and contexts never leak into the graph as fake actions. +Evaluation order is deliberately absent; ``linearize.py`` derives semantics +from this source structure in a separate stage. +""" + +from __future__ import annotations + +import ast +import bisect +import tokenize +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, replace +from enum import Enum +from functools import cached_property, lru_cache +from io import StringIO + + +class ForestBuildError(ValueError): + """The source cannot be represented by the current linear forest rung.""" + + +class ParameterRole(Enum): + """How an Action interprets the exact source carried by a Parameter.""" + + VALUE = "value" + TARGET = "target" + SUITE = "suite" + + +class AstDisposition(Enum): + """Total classification of Python AST classes, not a graph-node kind.""" + + ACTION = "action" + STRUCTURE = "structure" + METADATA = "metadata" + COMPOUND = "compound" + + +class NameAccess(Enum): + """Syntactic access performed by one exact Python ``Name`` occurrence.""" + + LOAD = "load" + STORE = "store" + DELETE = "delete" + + +@dataclass(frozen=True) +class SourceSpan: + """Absolute offsets and one-based line/column endpoints in source.""" + + start_offset: int + end_offset: int + start: tuple[int, int] + end: tuple[int, int] + + @property + def start_line(self) -> int: + """Return the one-based first source line.""" + + return self.start[0] + + @property + def start_column(self) -> int: + """Return the zero-based first source column.""" + + return self.start[1] + + @property + def end_line(self) -> int: + """Return the one-based final source line.""" + + return self.end[0] + + @property + def end_column(self) -> int: + """Return the zero-based exclusive final source column.""" + + return self.end[1] + + +@dataclass(frozen=True) +class NameOccurrence: + """One source-owned name span and its Python access context.""" + + name: str + access: NameAccess + span: SourceSpan + + +def _name_access(context: ast.expr_context, /) -> NameAccess: + """Map Python's closed name-context family to the public syntax record.""" + + if isinstance(context, ast.Load): + return NameAccess.LOAD + if isinstance(context, ast.Store): + return NameAccess.STORE + if isinstance(context, ast.Del): + return NameAccess.DELETE + raise ForestBuildError(f"unsupported Name context: {type(context).__name__}") + + +class _ModuleNameOccurrenceVisitor(ast.NodeVisitor): + """Select module-name occurrences without capturing comprehension locals.""" + + def __init__( + self, + index: _SourceIndex, + names: frozenset[str], + ) -> None: + self._index = index + self._names = names + self._bound: frozenset[str] = frozenset() + self.rows: list[NameOccurrence] = [] + + def visit_Name(self, node: ast.Name) -> None: + """Record one unshadowed module-name occurrence.""" + + if node.id in self._names and node.id not in self._bound: + self.rows.append( + NameOccurrence(node.id, _name_access(node.ctx), self._index.span(node)) + ) + + def visit_ListComp(self, node: ast.ListComp) -> None: + """Visit a list comprehension with Python's nested binding scope.""" + + self._visit_comprehension(node.generators, (node.elt,)) + + def visit_SetComp(self, node: ast.SetComp) -> None: + """Visit a set comprehension with Python's nested binding scope.""" + + self._visit_comprehension(node.generators, (node.elt,)) + + def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None: + """Visit a generator expression with Python's nested binding scope.""" + + self._visit_comprehension(node.generators, (node.elt,)) + + def visit_DictComp(self, node: ast.DictComp) -> None: + """Visit a dictionary comprehension with Python's nested binding scope.""" + + self._visit_comprehension(node.generators, (node.key, node.value)) + + def _visit_comprehension( + self, + generators: list[ast.comprehension], + outputs: tuple[ast.expr, ...], + ) -> None: + inherited = self._bound + current = set(inherited) + try: + for generator in generators: + self._bound = frozenset(current) + self.visit(generator.iter) + self._visit_target(generator.target) + current.update(_stored_target_names(generator.target)) + self._bound = frozenset(current) + for condition in generator.ifs: + self.visit(condition) + for output in outputs: + self.visit(output) + finally: + self._bound = inherited + + def _visit_target(self, node: ast.AST) -> None: + """Skip local Name stores but retain evaluated target subexpressions.""" + + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): + return + if isinstance(node, (ast.Tuple, ast.List, ast.Starred)): + for child in ast.iter_child_nodes(node): + self._visit_target(child) + return + self.visit(node) + + +class _PythonActivationBindingVisitor(ast.NodeVisitor): + """Collect binders encoded outside ordinary ``Name`` target nodes.""" + + def __init__(self) -> None: + self.names: set[str] = set() + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + """Record an exception alias and inspect its executable expressions.""" + + if node.name is not None: + self.names.add(node.name) + if node.type is not None: + self.visit(node.type) + for statement in node.body: + self.visit(statement) + + def visit_MatchAs(self, node: ast.MatchAs) -> None: + """Record an ``as`` or bare capture pattern.""" + + if node.name is not None: + self.names.add(node.name) + if node.pattern is not None: + self.visit(node.pattern) + + def visit_MatchStar(self, node: ast.MatchStar) -> None: + """Record a starred sequence-pattern capture.""" + + if node.name is not None: + self.names.add(node.name) + + def visit_MatchMapping(self, node: ast.MatchMapping) -> None: + """Record a mapping-rest capture and nested patterns.""" + + if node.rest is not None: + self.names.add(node.rest) + for key in node.keys: + self.visit(key) + for pattern in node.patterns: + self.visit(pattern) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Do not leak binders from an unexecuted nested function body.""" + + del node + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Do not leak binders from an unexecuted async function body.""" + + del node + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + """Do not leak binders from a nested class namespace.""" + + del node + + def visit_Lambda(self, node: ast.Lambda) -> None: + """Do not leak binders from a nested lambda scope.""" + + del node + + +def _stored_target_names(target: ast.AST, /) -> frozenset[str]: + """Return names bound in one comprehension target pattern.""" + + return frozenset( + node.id + for node in ast.walk(target) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) + ) + + +@dataclass(frozen=True) +class AliasValueFlow: + """Assignment targets receive exactly one existing binding value.""" + + targets: tuple[str, ...] + source: str + + +@dataclass(frozen=True) +class FreshValueFlow: + """Assignment targets share one fresh allocation and its initial contents.""" + + targets: tuple[str, ...] + type_name: str + contents: tuple[str, ...] + + +@dataclass(frozen=True) +class ContentLoadFlow: + """Assignment targets receive an object reachable through a container.""" + + targets: tuple[str, ...] + container: str + + +@dataclass(frozen=True) +class MethodCallFlow: + """Assignment targets receive the result of a receiver method call.""" + + targets: tuple[str, ...] + receiver: str + method: str + attributes: tuple[str, ...] + inputs: CallInputs + + +@dataclass(frozen=True) +class CallInput: + """Source-visible bindings that may contribute one call operand.""" + + bindings: tuple[str, ...] + + +@dataclass(frozen=True) +class CallInputs: + """Named call inputs without positional/keyword tuple conventions.""" + + positional: tuple[CallInput, ...] + keywords: tuple[CallInput, ...] + + def bindings(self) -> tuple[str, ...]: + """Return every visible binding once in canonical order.""" + + return tuple( + sorted( + { + binding + for item in (*self.positional, *self.keywords) + for binding in item.bindings + } + ) + ) + + +@dataclass(frozen=True) +class MethodEffect: + """One executed receiver call and its source-visible argument bindings.""" + + receiver: str + method: str + inputs: CallInputs + + +@dataclass(frozen=True) +class NamedCall: + """One direct name call whose builtin fallback can be certified.""" + + action_id: str + name: str + inputs: CallInputs + + +@dataclass(frozen=True) +class ReceiverCall: + """One direct binding receiver call eligible for exact type evidence.""" + + action_id: str + receiver: str + method: str + attributes: tuple[str, ...] + inputs: CallInputs + + +@dataclass(frozen=True) +class DynamicCall: + """A call whose callable expression has no exact statement-level shape.""" + + action_id: str + callable: CallInput + inputs: CallInputs + + +type SourceCall = NamedCall | ReceiverCall | DynamicCall + + +@dataclass(frozen=True, order=True) +class ImportBinding: + """One local import name and its exact qualified source target.""" + + name: str + qualified: str + + +@dataclass(frozen=True, order=True) +class ImportOccurrence: + """Exact import bindings established by one source Action.""" + + action_id: str + bindings: tuple[ImportBinding, ...] + + +@dataclass(frozen=True) +class UnknownValueFlow: + """Assignment result may be fresh or alias any source-visible input.""" + + targets: tuple[str, ...] + sources: tuple[str, ...] + + +type BindingValueFlow = ( + AliasValueFlow + | FreshValueFlow + | ContentLoadFlow + | MethodCallFlow + | UnknownValueFlow +) + + +@dataclass(frozen=True) +class SourceStatement: + """One top-level statement shape retained from the parsed source.""" + + root_id: str + kind: str + span: SourceSpan + nested_kinds: tuple[str, ...] + restrictions: tuple[str, ...] + reads: tuple[str, ...] + writes: tuple[str, ...] + in_place_writes: tuple[str, ...] + mutates: tuple[str, ...] + imports: tuple[str, ...] + import_bindings: tuple[ImportBinding, ...] + value_flows: tuple[BindingValueFlow, ...] + calls: tuple[SourceCall, ...] + method_effects: tuple[MethodEffect, ...] + + +@dataclass(frozen=True) +class SourceInventory: + """Typed source-admission input, independent of solver and renderer policy.""" + + statements: tuple[SourceStatement, ...] + import_occurrences: tuple[ImportOccurrence, ...] + + +@dataclass(frozen=True, order=True) +class TemplateHole: + """Where one child's source was removed from its owner's template. + + A hole is a **position**, never a byte pattern. The template around it is + arbitrary Python source in which `{0}`, `{value}`, and `{left}` are all + legal text — set displays, format strings, dict keys, comments — so a + filler that searched for those bytes would splice a child into text that + merely looked like a hole, and could even accept an ambiguous template + whose reconstruction happened to match. Offsets are in template + coordinates and never overlap. + """ + + start: int + end: int + name: str + + +#: One token of the source paired with the exact span it occupies. +type _TokenSpan = tuple[tokenize.TokenInfo, SourceSpan] + +#: Every token lying inside the indexed lines, in source order. +type _TokenSpans = tuple[_TokenSpan, ...] + +#: One child to cut out of its owner: where it sits and what names it. +type _ChildCut = tuple[SourceSpan, str] + +#: The holes of one template, ordered by position. +type TemplateHoles = tuple[TemplateHole, ...] + + +#: The source line each template line began on, one entry per template line. +type SourceLineNumbers = tuple[int, ...] + + +@dataclass(frozen=True) +class TemplateSource: + """One template, the positions its children were cut from, and its origin. + + The three travel together because none is usable alone: filling needs the + positions, the positions mean nothing without the text they index, and + relocating the text needs to know which source line each of its lines came + from. Any record that carries a template carries this instead of a bare + string, so a template can never reach a filler without the holes that belong + to it, nor a relocation without the origins it has to consult. + + `source_lines` exists because the obvious arithmetic is wrong. A template + line's source line is not its index plus the owner's first line: a multi-line + child is cut down to a one-line sentinel, so every line after the first such + hole sits earlier in the template than in the source. + """ + + template: str + holes: TemplateHoles + source_lines: SourceLineNumbers + + +@dataclass(frozen=True) +class Command: + """Executable statement-like Action with source-exact child holes.""" + + id: str + source_span: SourceSpan + source_text: str + source_template: TemplateSource + parameters: tuple[str, ...] + + @property + def template(self) -> str: + """The owner's source with each child replaced by a written sentinel.""" + + return self.source_template.template + + @property + def holes(self) -> TemplateHoles: + """Where each child was removed, in template coordinates.""" + + return self.source_template.holes + + +@dataclass(frozen=True) +class Expression: + """Executable expression Action with source-exact child holes.""" + + id: str + source_span: SourceSpan + source_text: str + source_template: TemplateSource + parameters: tuple[str, ...] + + @property + def template(self) -> str: + """The owner's source with each child replaced by a written sentinel.""" + + return self.source_template.template + + @property + def holes(self) -> TemplateHoles: + """Where each child was removed, in template coordinates.""" + + return self.source_template.holes + + +type Action = Command | Expression + +_PARAMETER_CHILD_TYPES: dict[ParameterRole, tuple[type[Action], ...]] = { + ParameterRole.VALUE: (Expression,), + ParameterRole.TARGET: (Expression,), + ParameterRole.SUITE: (Command, Expression), +} + + +@dataclass(frozen=True) +class _ParameterSource: + source_span: SourceSpan + source_text: str + source_template: TemplateSource + actions: tuple[str, ...] + + @property + def template(self) -> str: + """The owner's source with each child replaced by a written sentinel.""" + + return self.source_template.template + + @property + def holes(self) -> TemplateHoles: + """Where each child was removed, in template coordinates.""" + + return self.source_template.holes + + +@dataclass(frozen=True) +class Parameter: + """Non-executable source container owned by exactly one Action.""" + + id: str + owner: str + name: str + role: ParameterRole + source: _ParameterSource + + @property + def source_span(self) -> SourceSpan: + """Return the exact source extent represented by this Parameter.""" + + return self.source.source_span + + @property + def source_text(self) -> str: + """Return the original source represented by this Parameter.""" + + return self.source.source_text + + @property + def template(self) -> str: + """Return Parameter source with child Actions replaced by holes.""" + + return self.source.template + + @property + def actions(self) -> tuple[str, ...]: + """Return child Action IDs in source order.""" + + return self.source.actions + + +@dataclass(frozen=True) +class ActionForest: + """Immutable source-exact alternating graph of Actions and Parameters.""" + + source: str + actions: tuple[Action, ...] + parameters: tuple[Parameter, ...] + + @cached_property + def _actions_by_id(self) -> dict[str, Action]: + return {action.id: action for action in self.actions} + + @cached_property + def _source_builder(self) -> _Builder: + """Store the source-exact structural view for this immutable forest.""" + + builder = _Builder(self.source) + if builder.build() != self: + raise ForestBuildError("ActionForest cannot rebuild its exact source") + return builder + + @cached_property + def _parameters_by_id(self) -> dict[str, Parameter]: + return {parameter.id: parameter for parameter in self.parameters} + + @cached_property + def _parent_parameter_by_action(self) -> dict[str, str]: + return { + child_action: parameter.id + for parameter in self.parameters + for child_action in parameter.actions + } + + def action(self, action_id: str) -> Action: + """Resolve one source Action; execution contexts live outside the forest.""" + + return self._actions_by_id[action_id] + + def parameter(self, parameter_id: str) -> Parameter: + """Resolve one structural Parameter by its canonical ID.""" + + return self._parameters_by_id[parameter_id] + + def parent_action(self, action_id: str) -> str | None: + """Return the Action owning the Parameter that contains an Action.""" + + parameter_id = self.parent_parameter(action_id) + if parameter_id is None: + return None + owner = self.parameter(parameter_id).owner + return owner + + def parent_parameter(self, action_id: str) -> str | None: + """Return the unique structural Parameter containing this Action.""" + + return self._parent_parameter_by_action.get(action_id) + + def root_of(self, action_id: str) -> str: + """Return the cached outer source root owning one exact Action.""" + + try: + return self._root_by_action[action_id] + except KeyError as error: + raise KeyError(f"unknown Action {action_id!r}") from error + + @cached_property + def _root_by_action(self) -> dict[str, str]: + """Index structural ancestry once for all analysis consumers.""" + + result: dict[str, str] = {} + for action in self.actions: + path = [] + current = action.id + while current not in result: + path.append(current) + parent = self.parent_action(current) + if parent is None: + result[current] = current + break + current = parent + root = result[current] + result.update((item, root) for item in path) + return result + + @cached_property + def roots(self) -> tuple[str, ...]: + """Derive source roots from ownership; no StatementTree is stored.""" + + children = set(self._parent_parameter_by_action) + return tuple(action.id for action in self.actions if action.id not in children) + + def reconstruct_action(self, action_id: str) -> str: + """Rebuild one Action from its exact template and descendants.""" + + action = self.action(action_id) + values = { + self.parameter(parameter_id).name: self.reconstruct_parameter(parameter_id) + for parameter_id in action.parameters + } + return _fill(action.template, action.holes, values) + + def reconstruct_parameter(self, parameter_id: str) -> str: + """Rebuild one Parameter from its exact template and child Actions.""" + + parameter = self.parameter(parameter_id) + values = { + str(index): self.reconstruct_action(action_id) + for index, action_id in enumerate(parameter.actions) + } + return _fill(parameter.template, parameter.source.holes, values) + + def name_occurrences( + self, + action_id: str, + names: frozenset[str] | None = None, + /, + ) -> tuple[NameOccurrence, ...]: + """Return exact ``Name`` spans from the parsed AST.""" + + owner = self._source_builder.action_nodes[action_id] + rows = tuple( + NameOccurrence( + node.id, _name_access(node.ctx), self._source_builder.index.span(node) + ) + for node in ast.walk(owner) + if isinstance(node, ast.Name) and (names is None or node.id in names) + ) + return tuple(sorted(rows, key=lambda row: row.span.start_offset)) + + def module_name_occurrences( + self, + action_id: str, + names: frozenset[str], + /, + ) -> tuple[NameOccurrence, ...]: + """Return occurrences resolved to the surrounding module namespace.""" + + visitor = _ModuleNameOccurrenceVisitor(self._source_builder.index, names) + visitor.visit(self._source_builder.action_nodes[action_id]) + return tuple(sorted(visitor.rows, key=lambda row: row.span.start_offset)) + + def python_activation_bindings(self, action_id: str, /) -> frozenset[str]: + """Return syntax-owned binding names that cannot cross an activation.""" + + visitor = _PythonActivationBindingVisitor() + visitor.visit(self._source_builder.action_nodes[action_id]) + return frozenset(visitor.names) + + def project_action( + self, + action_id: str, + replacements: Mapping[SourceSpan, str], + /, + ) -> str: + """Project one Action by replacing validated, non-overlapping spans.""" + + action = self.action(action_id) + spans = tuple(sorted(replacements, key=lambda span: span.start_offset)) + if any( + span.start_offset < action.source_span.start_offset + or span.end_offset > action.source_span.end_offset + or span.start_offset >= span.end_offset + for span in spans + ): + raise ValueError("projected span lies outside its Action") + if any( + left.end_offset > right.start_offset + for left, right in zip(spans, spans[1:], strict=False) + ): + raise ValueError("projected Action spans overlap") + result = action.source_text + base = action.source_span.start_offset + for span in reversed(spans): + start = span.start_offset - base + end = span.end_offset - base + result = result[:start] + replacements[span] + result[end:] + return result + + def validate(self) -> None: + """Validate this immutable Forest once, then reuse its receipt.""" + + _ = self._validation_receipt + + @cached_property + def _validation_receipt(self) -> bool: + self._validate_uncached() + return True + + def _validate_uncached(self) -> None: + action_ids = _validate_forest_identifiers(self) + parent_count = dict.fromkeys(action_ids, 0) + _validate_action_parameters(self) + _validate_parameter_children(self, parent_count) + _validate_forest_roots(self, parent_count) + _validate_reconstruction(self) + + +@dataclass(frozen=True) +class BuiltSource: + """The forest and admission inventory produced by one shared AST parse.""" + + forest: ActionForest + inventory: SourceInventory + + def statement(self, root_id: str) -> SourceStatement: + """Resolve one top-level statement through the source-owned index.""" + + try: + return self._statements_by_root[root_id] + except KeyError as error: + raise KeyError(f"unknown statement root {root_id!r}") from error + + @cached_property + def _statements_by_root(self) -> dict[str, SourceStatement]: + """Index immutable statement summaries once for all consumers.""" + + return {statement.root_id: statement for statement in self.inventory.statements} + + +def _validate_forest_identifiers(forest: ActionForest) -> tuple[str, ...]: + action_ids = tuple(action.id for action in forest.actions) + parameter_ids = tuple(parameter.id for parameter in forest.parameters) + checks = ( + (len(action_ids) != len(set(action_ids)), "Action identifiers are not unique"), + ( + len(parameter_ids) != len(set(parameter_ids)), + "Parameter identifiers are not unique", + ), + ( + bool(set(action_ids) & set(parameter_ids)), + "Action and Parameter identifiers overlap", + ), + ) + failure = next((message for failed, message in checks if failed), None) + if failure is not None: + raise ForestBuildError(failure) + return action_ids + + +def _validate_action_parameters(forest: ActionForest) -> None: + for action in forest.actions: + _validate_placeholders( + action.holes, + {forest.parameter(item).name for item in action.parameters}, + ) + for parameter_id in action.parameters: + if forest.parameter(parameter_id).owner != action.id: + raise ForestBuildError("Parameter owner does not match its Action") + + +def _validate_parameter_children( + forest: ActionForest, parent_count: dict[str, int] +) -> None: + for parameter in forest.parameters: + if not isinstance(parameter.role, ParameterRole): + raise ForestBuildError("Parameter role is invalid") + _validate_placeholders( + parameter.source.holes, + {str(index) for index in range(len(parameter.actions))}, + ) + expected_child_types = _PARAMETER_CHILD_TYPES[parameter.role] + for action_id in parameter.actions: + if not isinstance(forest.action(action_id), expected_child_types): + raise ForestBuildError("Parameter child type does not match its role") + parent_count[action_id] += 1 + + +def _validate_forest_roots( + forest: ActionForest, parent_count: Mapping[str, int] +) -> None: + roots = set(forest.roots) + for action_id, count in parent_count.items(): + expected = 0 if action_id in roots else 1 + if count != expected: + raise ForestBuildError("Action/Parameter alternation is not a forest") + + +def _validate_reconstruction(forest: ActionForest) -> None: + reconstructed_actions: dict[str, str] = {} + reconstructed_parameters: dict[str, str] = {} + + def reconstruct_action(action_id: str) -> str: + cached = reconstructed_actions.get(action_id) + if cached is not None: + return cached + action = forest.action(action_id) + result = _fill( + action.template, + action.holes, + { + forest.parameter(parameter_id).name: reconstruct_parameter(parameter_id) + for parameter_id in action.parameters + }, + ) + reconstructed_actions[action_id] = result + return result + + def reconstruct_parameter(parameter_id: str) -> str: + cached = reconstructed_parameters.get(parameter_id) + if cached is not None: + return cached + parameter = forest.parameter(parameter_id) + result = _fill( + parameter.template, + parameter.source.holes, + { + str(index): reconstruct_action(action_id) + for index, action_id in enumerate(parameter.actions) + }, + ) + reconstructed_parameters[parameter_id] = result + return result + + if any( + reconstruct_action(action.id) != action.source_text for action in forest.actions + ): + raise ForestBuildError("Action template is not source-exact") + if any( + reconstruct_parameter(parameter.id) != parameter.source_text + for parameter in forest.parameters + ): + raise ForestBuildError("Parameter template is not source-exact") + + +_COMPOUND_STATEMENTS = tuple( + item + for item in ( + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.ClassDef, + ast.For, + ast.AsyncFor, + ast.While, + ast.If, + ast.With, + ast.AsyncWith, + ast.Match, + ast.Try, + getattr(ast, "TryStar", None), + ) + if item is not None +) + +_FIELD_NAMES: dict[type[ast.AST], dict[str, str]] = { + ast.FunctionDef: { + "name": "target", + "args": "signature", + "body": "body", + "decorator_list": "decorator", + "returns": "return_annotation", + }, + ast.ClassDef: { + "name": "target", + "bases": "base", + "keywords": "keyword", + "body": "body", + "decorator_list": "decorator", + }, + ast.arguments: { + "posonlyargs": "annotation", + "args": "annotation", + "vararg": "annotation", + "kwonlyargs": "annotation", + "kw_defaults": "default", + "kwarg": "annotation", + "defaults": "default", + }, + ast.arg: {"annotation": "value"}, + ast.If: {"test": "condition", "body": "body", "orelse": "orelse"}, + ast.While: {"test": "condition", "body": "body", "orelse": "orelse"}, + ast.For: { + "target": "target", + "iter": "iterable", + "body": "body", + "orelse": "orelse", + }, + ast.With: {"items": "item", "body": "body"}, + ast.Try: { + "body": "body", + "handlers": "handler", + "orelse": "orelse", + "finalbody": "finalbody", + }, + ast.Match: {"subject": "subject", "cases": "case"}, + ast.match_case: { + "pattern": "pattern", + "guard": "guard", + "body": "body", + }, + ast.withitem: {"context_expr": "context", "optional_vars": "target"}, + ast.ExceptHandler: {"type": "type", "name": "target", "body": "body"}, + ast.Assign: {"targets": "target", "value": "value"}, + ast.AnnAssign: { + "target": "target", + "annotation": "annotation", + "value": "value", + }, + ast.AugAssign: {"target": "target", "value": "value"}, + ast.NamedExpr: {"target": "target", "value": "value"}, + ast.BinOp: {"left": "left", "right": "right"}, + ast.BoolOp: {"values": "operand"}, + ast.UnaryOp: {"operand": "operand"}, + ast.IfExp: {"body": "then", "test": "condition", "orelse": "otherwise"}, + ast.Call: {"func": "callable", "args": "positional", "keywords": "keyword"}, + ast.Attribute: {"value": "base"}, + ast.Subscript: {"value": "base", "slice": "index"}, + ast.Compare: {"left": "left", "comparators": "comparator"}, + ast.Lambda: {"args": "parameters", "body": "body"}, + ast.Return: {"value": "value"}, + ast.Delete: {"targets": "target"}, + ast.Raise: {"exc": "error", "cause": "cause"}, + ast.Assert: {"test": "condition", "msg": "message"}, + ast.Import: {"names": "module"}, + ast.ImportFrom: {"names": "name"}, + ast.List: {"elts": "element"}, + ast.Tuple: {"elts": "element"}, + ast.Set: {"elts": "element"}, + ast.Dict: {"keys": "key", "values": "value"}, + ast.ListComp: {"elt": "element", "generators": "generator"}, + ast.SetComp: {"elt": "element", "generators": "generator"}, + ast.GeneratorExp: {"elt": "element", "generators": "generator"}, + ast.DictComp: { + "key": "key", + "value": "value", + "generators": "generator", + }, + ast.Yield: {"value": "value"}, + ast.YieldFrom: {"value": "value"}, + ast.Await: {"value": "value"}, + ast.JoinedStr: {"values": "part"}, + ast.Slice: {"lower": "lower", "upper": "upper", "step": "step"}, +} + +_ATOMIC_ACTIONS = ( + ast.Constant, + ast.Name, + ast.Pass, + ast.Break, + ast.Continue, + ast.Global, + ast.Nonlocal, +) + + +def ast_disposition(node_type: type[ast.AST]) -> AstDisposition: + """Classify every concrete Python AST class without silently dropping one.""" + + if issubclass(node_type, _COMPOUND_STATEMENTS): + return AstDisposition.COMPOUND + if issubclass(node_type, ast.expr): + return AstDisposition.ACTION + if issubclass(node_type, ast.stmt): + return AstDisposition.ACTION + if issubclass( + node_type, + (ast.operator, ast.unaryop, ast.boolop, ast.cmpop, ast.expr_context), + ): + return AstDisposition.METADATA + return AstDisposition.STRUCTURE + + +def concrete_ast_classes() -> tuple[type[ast.AST], ...]: + """Return all concrete AST classes shipped by the active Python runtime.""" + + pending = [ast.AST] + found: set[type[ast.AST]] = set() + while pending: + parent = pending.pop() + for child in parent.__subclasses__(): + if child not in found: + found.add(child) + pending.append(child) + return tuple(sorted(found, key=lambda item: item.__name__)) + + +@lru_cache(maxsize=1) +def assert_total_ast_classification() -> None: + """Fail if the running Python exposes an unclassified concrete AST type.""" + + for node_type in concrete_ast_classes(): + disposition = ast_disposition(node_type) + if not isinstance(disposition, AstDisposition): + raise ForestBuildError(f"unclassified AST class: {node_type.__name__}") + + +@dataclass(frozen=True) +class _SourceOccurrence: + source_span: SourceSpan + role: ParameterRole + + +@dataclass(frozen=True) +class _Component: + name: str + node: ast.AST | _SourceOccurrence + origin_field: str + origin_index: int + suite: tuple[ast.stmt, ...] = () + + @property + def role(self) -> ParameterRole: + """Return the structural role contributed by this component.""" + + if self.suite: + return ParameterRole.SUITE + if isinstance(self.node, _SourceOccurrence): + return self.node.role + return _parameter_role(self.node) + + @property + def child_nodes(self) -> tuple[ast.AST, ...]: + """Return executable child nodes owned by this component.""" + + if self.suite: + return tuple(_statement_action_root(statement) for statement in self.suite) + if isinstance(self.node, _SourceOccurrence): + return () + return _parameter_action_roots(self.node) + + +@dataclass(frozen=True) +class _CompoundProtocol: + fields: tuple[str, ...] + suites: frozenset[str] + + +_IF_OR_WHILE_PROTOCOL = _CompoundProtocol( + fields=("test", "body", "orelse"), + suites=frozenset({"body", "orelse"}), +) +_FOR_PROTOCOL = _CompoundProtocol( + fields=("target", "iter", "body", "orelse"), + suites=frozenset({"body", "orelse"}), +) +_WITH_PROTOCOL = _CompoundProtocol( + fields=("items", "body"), + suites=frozenset({"body"}), +) +_TRY_PROTOCOL = _CompoundProtocol( + fields=("body", "handlers", "orelse", "finalbody"), + suites=frozenset({"body", "orelse", "finalbody"}), +) +_HANDLER_PROTOCOL = _CompoundProtocol( + fields=("type", "name", "body"), + suites=frozenset({"body"}), +) +_FUNCTION_PROTOCOL = _CompoundProtocol( + fields=("name", "args", "body", "decorator_list", "returns"), + suites=frozenset({"body"}), +) +_CLASS_PROTOCOL = _CompoundProtocol( + fields=("name", "bases", "keywords", "body", "decorator_list"), + suites=frozenset({"body"}), +) +_MATCH_PROTOCOL = _CompoundProtocol( + fields=("subject", "cases"), + suites=frozenset(), +) +_MATCH_CASE_PROTOCOL = _CompoundProtocol( + fields=("pattern", "guard", "body"), + suites=frozenset({"body"}), +) +_ARGUMENT_PROTOCOL = _CompoundProtocol( + fields=("annotation",), + suites=frozenset(), +) +_COMPOUND_PROTOCOLS: dict[type[ast.AST], _CompoundProtocol] = { + ast.If: _IF_OR_WHILE_PROTOCOL, + ast.While: _IF_OR_WHILE_PROTOCOL, + ast.For: _FOR_PROTOCOL, + ast.With: _WITH_PROTOCOL, + ast.Try: _TRY_PROTOCOL, + ast.TryStar: _TRY_PROTOCOL, + ast.Match: _MATCH_PROTOCOL, + ast.FunctionDef: _FUNCTION_PROTOCOL, + ast.ClassDef: _CLASS_PROTOCOL, +} +_STRUCTURE_PROTOCOLS: dict[type[ast.AST], _CompoundProtocol] = { + ast.ExceptHandler: _HANDLER_PROTOCOL, + ast.match_case: _MATCH_CASE_PROTOCOL, + ast.arg: _ARGUMENT_PROTOCOL, +} + +_OWNER_PREFIX_FIELDS: dict[type[ast.AST], tuple[str, ...]] = { + ast.FunctionDef: ("decorator_list",), + ast.ClassDef: ("decorator_list",), +} +_UNSUPPORTED_NONEMPTY_FIELDS = { + (ast.FunctionDef, "type_params"), + (ast.ClassDef, "type_params"), +} + + +class _SourceIndex: + def __init__(self, source: str) -> None: + self.source = source + self._lines = source.splitlines(keepends=True) + self._line_is_ascii = tuple(line.isascii() for line in self._lines) + self._line_offsets: list[int] = [] + self._spans: dict[ast.AST, SourceSpan] = {} + self._tokens = tuple(tokenize.generate_tokens(StringIO(source).readline)) + offset = 0 + for line in self._lines: + self._line_offsets.append(offset) + offset += len(line) + self._indexed_tokens = self._index_tokens() + self._token_starts = tuple( + span.start_offset for _token, span in self._indexed_tokens + ) + + def _index_tokens(self) -> _TokenSpans: + """Span every token once, so no later lookup recomputes one. + + Tokens positioned past the last indexed line are the terminators + `tokenize` appends; they have no source extent, so they are dropped here + rather than guarded against at each use. + """ + + return tuple( + (token, self._token_span(token)) + for token in self._tokens + if token.start[0] <= len(self._lines) and token.end[0] <= len(self._lines) + ) + + def span(self, node: ast.AST) -> SourceSpan: + """The one source extent of a node, decorators included. + + A decorated `def` or `class` owns its decorators: they are part of the + statement's text, not of whatever contains it. This is the + span authority. A second, decorator-excluding one used to exist for + holes and suite extents, so a decorated definition inside any suite had + its decorator both left in the template and returned by the child, + duplicating it in the reconstruction. + """ + + cached = self._spans.get(node) + if cached is not None: + return cached + if not _has_span(node): + raise ForestBuildError(f"AST node {type(node).__name__} has no source span") + result = self._prefixed_span(node, self._bare_span(node)) + self._spans[node] = result + return result + + def _bare_span(self, node: ast.AST) -> SourceSpan: + """The node's own extent, before its owned prefixes are folded in.""" + + start = (node.lineno, node.col_offset) + end = (node.end_lineno, node.end_col_offset) + return SourceSpan( + start_offset=self._offset(*start), + end_offset=self._offset(*end), + start=start, + end=end, + ) + + def _prefixed_span(self, node: ast.AST, span: SourceSpan) -> SourceSpan: + """Extend an extent backwards over the prefixes the node owns.""" + + prefixes = tuple( + item + for field_name in _OWNER_PREFIX_FIELDS.get(type(node), ()) + for item in getattr(node, field_name, ()) + if isinstance(item, ast.AST) + ) + if not prefixes: + return span + first = min(prefixes, key=_position) + start = (first.lineno, node.col_offset) + return SourceSpan( + start_offset=self._offset(*start), + end_offset=span.end_offset, + start=start, + end=span.end, + ) + + def text(self, span: SourceSpan) -> str: + """Return the exact source text covered by one canonical span.""" + + return self.source[span.start_offset : span.end_offset] + + def covering_span(self, nodes: tuple[ast.AST, ...]) -> SourceSpan: + """Return the minimal span covering a nonempty ordered node sequence.""" + + first = self.span(nodes[0]) + last = self.span(nodes[-1]) + return SourceSpan( + start_offset=first.start_offset, + end_offset=last.end_offset, + start=first.start, + end=last.end, + ) + + def marked_name( + self, + owner: ast.AST, + marker: str, + name: str, + role: ParameterRole, + ) -> _SourceOccurrence: + """Locate the name a marker keyword introduces inside one owner. + + Reads pre-computed token spans over the owner's own window. The earlier + version spanned every token in the file on each call, which made a forest + build quadratic in its token count and, since the renderer builds a + forest of each generated driver, dominated rendering entirely. + """ + + window = self._tokens_inside(self.span(owner)) + for (marker_token, _span), (target, target_span) in zip( + window, window[1:], strict=False + ): + if ( + marker_token.type == tokenize.NAME + and marker_token.string == marker + and target.type == tokenize.NAME + and target.string == name + ): + return _SourceOccurrence(source_span=target_span, role=role) + raise ForestBuildError(f"{marker} target has no source token") + + def _tokens_inside(self, span: SourceSpan) -> _TokenSpans: + """The indexed tokens contained in one span, located rather than scanned. + + Token starts ascend, so the candidate window is found by bisection. The + containment test still runs, because a token starting inside the span may + end outside it. + """ + + first = bisect.bisect_left(self._token_starts, span.start_offset) + last = bisect.bisect_right(self._token_starts, span.end_offset) + return tuple( + row + for row in self._indexed_tokens[first:last] + if _span_is_inside(row[1], span) + ) + + def _token_span(self, token: tokenize.TokenInfo) -> SourceSpan: + start_line, start_column = token.start + end_line, end_column = token.end + start_byte_column = len( + self._lines[start_line - 1][:start_column].encode("utf-8") + ) + end_byte_column = len(self._lines[end_line - 1][:end_column].encode("utf-8")) + return SourceSpan( + start_offset=self._offset(start_line, start_byte_column), + end_offset=self._offset(end_line, end_byte_column), + start=(start_line, start_byte_column), + end=(end_line, end_byte_column), + ) + + def _offset(self, line_number: int, utf8_column: int) -> int: + if self._line_is_ascii[line_number - 1]: + return self._line_offsets[line_number - 1] + utf8_column + line = self._lines[line_number - 1] + prefix = line.encode("utf-8")[:utf8_column].decode("utf-8") + return self._line_offsets[line_number - 1] + len(prefix) + + +class _Builder: + def __init__(self, source: str) -> None: + self.source = source + self.index = _SourceIndex(source) + self.module = ast.parse(source, type_comments=True) + self.actions: list[Action] = [] + self.parameters: list[Parameter] = [] + self.action_nodes: dict[str, ast.AST] = {} + self.components_by_action: dict[str, tuple[_Component, ...]] = {} + + def build(self) -> ActionForest: + """Build and validate the forest from the module parsed at construction.""" + + assert_total_ast_classification() + for statement_number, statement in enumerate(self.module.body, start=1): + root_node = _statement_action_root(statement) + root_id = f"s{statement_number}" + self._build_action(root_node, root_id, root=True) + forest = ActionForest( + source=self.source, + actions=tuple(self.actions), + parameters=tuple(self.parameters), + ) + forest.validate() + object.__setattr__(forest, "_source_builder", self) + return forest + + def inventory(self) -> SourceInventory: + """Project admission facts from the already parsed module.""" + + action_by_node = { + id(node): action_id for action_id, node in self.action_nodes.items() + } + statements = tuple( + _source_statement(self.index, statement, index, action_by_node) + for index, statement in enumerate(self.module.body, start=1) + ) + occurrences = tuple( + ImportOccurrence(action_id, bindings) + for _offset, action_id, bindings in sorted( + ( + self.index.span(node).start_offset, + action_id, + import_bindings(node), + ) + for action_id, node in self.action_nodes.items() + if isinstance(node, (ast.Import, ast.ImportFrom)) + ) + if bindings + ) + return SourceInventory(statements, occurrences) + + def _build_action(self, node: ast.AST, action_id: str, *, root: bool) -> str: + disposition = ast_disposition(type(node)) + if ( + disposition is AstDisposition.COMPOUND + and type(node) not in _COMPOUND_PROTOCOLS + ): + raise ForestBuildError( + f"unsupported compound statement: {type(node).__name__}" + ) + if disposition not in {AstDisposition.ACTION, AstDisposition.COMPOUND}: + raise ForestBuildError( + f"{type(node).__name__} cannot be an executable Action" + ) + span = self.index.span(node) + components = _components(node, self.index) + self.action_nodes[action_id] = node + self.components_by_action[action_id] = components + parameter_ids: list[str] = [] + replacements: list[_ChildCut] = [] + for component_number, component in enumerate(components): + name = _unique_component_name(components, component_number) + parameter_id = f"{action_id}.{name}" + parameter = self._build_parameter( + replace(component, name=name), + parameter_id, + owner=action_id, + ) + self.parameters.append(parameter) + parameter_ids.append(parameter_id) + replacements.append((parameter.source_span, name)) + source_text = self.index.text(span) + cut = _replace_source(source_text, span, replacements) + common_fields = { + "id": action_id, + "source_span": span, + "source_text": source_text, + "source_template": cut, + "parameters": tuple(parameter_ids), + } + action: Action + if root and isinstance(node, ast.stmt): + action = Command(**common_fields) + else: + action = Expression(**common_fields) + # Parents are stored before descendants for deterministic readable order. + insertion = len(self.actions) + self.actions.insert(insertion, action) + return action_id + + def _build_parameter( + self, + component: _Component, + parameter_id: str, + *, + owner: str, + ) -> Parameter: + span = _component_span(component, self.index) + action_ids: list[str] = [] + replacements: list[_ChildCut] = [] + owner_node = self.action_nodes[owner] + child_nodes = _parameter_action_children(component, owner_node) + for child_number, child in enumerate(child_nodes): + child_id = f"{parameter_id}.{child_number}" + self._build_action(child, child_id, root=bool(component.suite)) + action_ids.append(child_id) + replacements.append((self.index.span(child), str(child_number))) + source_text = self.index.text(span) + cut = _replace_source(source_text, span, replacements) + return Parameter( + id=parameter_id, + owner=owner, + name=component.name, + role=component.role, + source=_ParameterSource( + source_span=span, + source_text=source_text, + source_template=cut, + actions=tuple(action_ids), + ), + ) + + +def build_forest(source: str) -> ActionForest: + """Build and validate a source-exact ActionForest from one source string.""" + + return _Builder(source).build() + + +def build_forest_with_inventory(source: str) -> BuiltSource: + """Build source structure and admission facts without parsing twice.""" + + builder = _Builder(source) + forest = builder.build() + return BuiltSource(forest, builder.inventory()) + + +def _source_statement( + index: _SourceIndex, + statement: ast.stmt, + number: int, + action_by_node: Mapping[int, str], +) -> SourceStatement: + reads, writes, in_place_writes, mutates = _statement_bindings(statement) + calls, method_effects = _call_facts(statement, action_by_node) + return SourceStatement( + root_id=f"s{number}", + kind=type(statement).__name__, + span=index.span(statement), + nested_kinds=tuple( + sorted( + { + type(node).__name__ + for node in ast.walk(statement) + if node is not statement + } + ) + ), + restrictions=_statement_restrictions(statement), + reads=reads, + writes=writes, + in_place_writes=in_place_writes, + mutates=mutates, + imports=_imported_bindings(statement), + import_bindings=import_bindings(statement), + value_flows=_binding_value_flows(statement), + calls=calls, + method_effects=method_effects, + ) + + +def _statement_bindings( + statement: ast.stmt, +) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + visitor = _BindingVisitor() + visitor.visit(statement) + reads = (visitor.reads - visitor.writes) | visitor.read_before_write + return ( + tuple(sorted(reads | visitor.mutates)), + tuple(sorted(visitor.writes)), + tuple(sorted(visitor.read_before_write & visitor.writes)), + tuple(sorted(visitor.mutates)), + ) + + +class _BindingVisitor(ast.NodeVisitor): + """Project one opaque statement to conservative module binding facts.""" + + def __init__(self) -> None: + self.reads: set[str] = set() + self.writes: set[str] = set() + self.read_before_write: set[str] = set() + self.mutates: set[str] = set() + + def visit_Name(self, node: ast.Name) -> None: + """Record module-level reads, stores, and deletions.""" + + if isinstance(node.ctx, ast.Load): + self.reads.add(node.id) + elif isinstance(node.ctx, (ast.Store, ast.Del)): + self.writes.add(node.id) + + def visit_AugAssign(self, node: ast.AugAssign) -> None: + """Record augmented-assignment read-before-write and mutation semantics.""" + + if isinstance(node.target, ast.Name): + self.read_before_write.add(node.target.id) + self.writes.add(node.target.id) + else: + self.visit(node.target) + if (name := _target_base_name(node.target)) is not None: + self.mutates.add(name) + self.visit(node.value) + + def visit_Assign(self, node: ast.Assign) -> None: + """Record assignment value reads, writes, and non-name mutations.""" + + self.visit(node.value) + for target in node.targets: + self.visit(target) + if ( + not isinstance(target, (ast.Name, ast.Tuple, ast.List)) + and (name := _target_base_name(target)) is not None + ): + self.mutates.add(name) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + """Record annotated assignment dependencies and target mutation.""" + + self.visit(node.target) + self.visit(node.annotation) + if node.value is not None: + self.visit(node.value) + if ( + not isinstance(node.target, ast.Name) + and (name := _target_base_name(node.target)) is not None + ): + self.mutates.add(name) + + def visit_Call(self, node: ast.Call) -> None: + """Record call reads; aliasing owns certified versus unknown effects.""" + + self.generic_visit(node) + + def visit_Import(self, node: ast.Import) -> None: + """Record names established by an import statement.""" + + self.writes.update( + alias.asname or alias.name.partition(".")[0] for alias in node.names + ) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + """Record non-star names established by a from-import statement.""" + + self.writes.update( + alias.asname or alias.name for alias in node.names if alias.name != "*" + ) + + def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None: + """Treat the conditionally bound and cleared exception alias as a write.""" + + if node.name is not None: + self.writes.add(node.name) + if node.type is not None: + self.visit(node.type) + for statement in node.body: + self.visit(statement) + + def visit_MatchAs(self, node: ast.MatchAs) -> None: + """Record an ``as`` or bare pattern capture as a conditional write.""" + + if node.name is not None: + self.writes.add(node.name) + if node.pattern is not None: + self.visit(node.pattern) + + def visit_MatchStar(self, node: ast.MatchStar) -> None: + """Record a starred sequence-pattern capture as a conditional write.""" + + if node.name is not None: + self.writes.add(node.name) + + def visit_MatchMapping(self, node: ast.MatchMapping) -> None: + """Record a mapping-rest capture and visit nested expressions.""" + + if node.rest is not None: + self.writes.add(node.rest) + for key in node.keys: + self.visit(key) + for pattern in node.patterns: + self.visit(pattern) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Record a synchronous definition without executing its body.""" + + self._visit_function(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Record an asynchronous definition without executing its body.""" + + self._visit_function(node) + + def _visit_function( + self, + node: ast.FunctionDef | ast.AsyncFunctionDef, + ) -> None: + self.writes.add(node.name) + for expression in ( + *node.decorator_list, + *node.args.defaults, + *(value for value in node.args.kw_defaults if value is not None), + ): + self.visit(expression) + if node.returns is not None: + self.visit(node.returns) + scope = _BindingVisitor() + scope.writes.update(argument.arg for argument in _arguments(node.args)) + for statement in node.body: + scope.visit(statement) + self.reads.update((scope.reads - scope.writes) | scope.read_before_write) + + def visit_Lambda(self, node: ast.Lambda) -> None: + """Project only free reads from a dormant lambda body.""" + + scope = _BindingVisitor() + scope.writes.update(argument.arg for argument in _arguments(node.args)) + scope.visit(node.body) + self.reads.update((scope.reads - scope.writes) | scope.read_before_write) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + """Record class creation and module reads from its executed definition.""" + + self.writes.add(node.name) + for expression in (*node.decorator_list, *node.bases): + self.visit(expression) + for keyword in node.keywords: + self.visit(keyword.value) + scope = _BindingVisitor() + for statement in node.body: + scope.visit(statement) + self.reads.update((scope.reads - scope.writes) | scope.read_before_write) + + +def _arguments(arguments: ast.arguments) -> tuple[ast.arg, ...]: + positional = (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + optional = tuple( + argument + for argument in (arguments.vararg, arguments.kwarg) + if argument is not None + ) + return (*positional, *optional) + + +def _target_base_name(node: ast.AST) -> str | None: + current = node + while isinstance(current, (ast.Attribute, ast.Subscript)): + current = current.value + return current.id if isinstance(current, ast.Name) else None + + +def _binding_value_flows(statement: ast.stmt) -> tuple[BindingValueFlow, ...]: + """Project executed binding flows without changing statement granularity.""" + + assignment = _outer_assignment(statement) + if assignment is not None: + return _assignment_value_flow(*assignment) + if not isinstance( + statement, + ( + ast.AsyncFor, + ast.AsyncWith, + ast.For, + ast.If, + ast.Match, + ast.Try, + ast.TryStar, + ast.While, + ast.With, + ), + ): + return () + visitor = _OpaqueStatementFlowVisitor() + visitor.visit(statement) + return tuple(visitor.flows) + + +def _assignment_value_flow( + targets: tuple[ast.expr, ...], + value: ast.expr, +) -> tuple[BindingValueFlow, ...]: + """Describe one assignment using the closed points-to vocabulary.""" + + names = tuple(target.id for target in targets if isinstance(target, ast.Name)) + if not names: + return () + match value: + case ast.Name(id=source): + return (AliasValueFlow(names, source),) + case ast.List() | ast.Tuple() | ast.Set() | ast.Dict(): + return ( + FreshValueFlow( + names, + f"builtins:{type(value).__name__.lower()}", + _loaded_names(value), + ), + ) + case ast.Subscript(value=container_node) if ( + container := _target_base_name(container_node) + ) is not None: + return (ContentLoadFlow(names, container),) + case ast.Call(func=ast.Attribute(value=receiver_node, attr=method)) if ( + receiver := _target_base_name(receiver_node) + ) is not None: + attribute_call = _attribute_chain(value.func) + if attribute_call is None: + return (UnknownValueFlow(names, _loaded_names(value)),) + receiver, attributes = attribute_call + return ( + MethodCallFlow( + names, + receiver, + method, + attributes, + _call_inputs(value), + ), + ) + case _: + return (UnknownValueFlow(names, _loaded_names(value)),) + + +class _OpaqueStatementFlowVisitor(ast.NodeVisitor): + """Collect may-flows executed inside one opaque top-level statement.""" + + def __init__(self) -> None: + self.flows: list[BindingValueFlow] = [] + + def visit_Assign(self, node: ast.Assign) -> None: + """Collect a may-flow for an assignment executed by opaque control.""" + + self.flows.extend(_assignment_value_flow(tuple(node.targets), node.value)) + self.visit(node.value) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + """Collect a may-flow for an executed annotated assignment.""" + + if node.value is not None: + self.flows.extend(_assignment_value_flow((node.target,), node.value)) + self.visit(node.value) + + def visit_NamedExpr(self, node: ast.NamedExpr) -> None: + """Collect a may-flow for an executed assignment expression.""" + + self.flows.extend(_assignment_value_flow((node.target,), node.value)) + self.visit(node.value) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Visit definition-time expressions while keeping the body dormant.""" + + self._visit_definition_expressions(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Visit async definition-time expressions while keeping the body dormant.""" + + self._visit_definition_expressions(node) + + def _visit_definition_expressions( + self, + node: ast.FunctionDef | ast.AsyncFunctionDef, + ) -> None: + for expression in ( + *node.decorator_list, + *node.args.defaults, + *(value for value in node.args.kw_defaults if value is not None), + ): + self.visit(expression) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + """Visit class bases and decorators without entering its local namespace.""" + + for expression in (*node.decorator_list, *node.bases): + self.visit(expression) + for keyword in node.keywords: + self.visit(keyword.value) + + def visit_Lambda(self, node: ast.Lambda) -> None: + """Visit lambda defaults without treating its dormant body as executed.""" + + for expression in ( + *node.args.defaults, + *(value for value in node.args.kw_defaults if value is not None), + ): + self.visit(expression) + + +def _outer_assignment( + statement: ast.stmt, +) -> tuple[tuple[ast.expr, ...], ast.expr] | None: + if isinstance(statement, ast.Assign): + return tuple(statement.targets), statement.value + if isinstance(statement, ast.AnnAssign) and statement.value is not None: + return (statement.target,), statement.value + return None + + +def _loaded_names(node: ast.AST) -> tuple[str, ...]: + return tuple( + sorted( + { + child.id + for child in ast.walk(node) + if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Load) + } + ) + ) + + +def _call_facts( + statement: ast.stmt, + action_by_node: Mapping[int, str], +) -> tuple[tuple[SourceCall, ...], tuple[MethodEffect, ...]]: + """Collect call shape and method transfer facts in one source walk.""" + + visitor = _CallFactVisitor(action_by_node) + visitor.visit(statement) + return tuple(visitor.calls), tuple(visitor.effects) + + +class _CallFactVisitor(ast.NodeVisitor): + """Collect executed call facts while leaving function bodies dormant.""" + + def __init__(self, action_by_node: Mapping[int, str]) -> None: + self.action_by_node = action_by_node + self.calls: list[SourceCall] = [] + self.effects: list[MethodEffect] = [] + + def visit_Call(self, node: ast.Call) -> None: + """Record one call using closed statement-level shape variants.""" + + inputs = _call_inputs(node) + try: + action_id = self.action_by_node[id(node)] + except KeyError as error: + raise ForestBuildError("call fact has no source Action") from error + call = _source_call(action_id, node.func, inputs) + self.calls.append(call) + if isinstance(call, ReceiverCall) and len(call.attributes) == 1: + self.effects.append(MethodEffect(call.receiver, call.method, inputs)) + self.generic_visit(node) + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + """Inspect synchronous definition-time calls but not its body.""" + + self._visit_function_definition(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + """Inspect asynchronous definition-time calls but not its body.""" + + self._visit_function_definition(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: + """Inspect lambda defaults while leaving its deferred body dormant.""" + + for expression in ( + *node.args.defaults, + *(value for value in node.args.kw_defaults if value is not None), + ): + self.visit(expression) + + def _visit_function_definition( + self, + node: ast.FunctionDef | ast.AsyncFunctionDef, + ) -> None: + for expression in ( + *node.decorator_list, + *node.args.defaults, + *(value for value in node.args.kw_defaults if value is not None), + ): + self.visit(expression) + + +def _call_inputs(node: ast.Call) -> CallInputs: + """Project positional and keyword operands without tuple role conventions.""" + + return CallInputs( + tuple(CallInput(_loaded_names(argument)) for argument in node.args), + tuple(CallInput(_loaded_names(keyword.value)) for keyword in node.keywords), + ) + + +def _source_call( + action_id: str, + function: ast.expr, + inputs: CallInputs, +) -> SourceCall: + """Classify one callable expression without resolving runtime identity.""" + + if isinstance(function, ast.Name): + return NamedCall(action_id, function.id, inputs) + attribute_call = _attribute_chain(function) + if attribute_call is not None: + receiver, attributes = attribute_call + return ReceiverCall( + action_id, + receiver, + attributes[-1], + attributes, + inputs, + ) + return DynamicCall(action_id, CallInput(_loaded_names(function)), inputs) + + +def _attribute_chain(node: ast.expr) -> tuple[str, tuple[str, ...]] | None: + """Return a name-rooted attribute path without resolving its identity.""" + + attributes = [] + current = node + while isinstance(current, ast.Attribute): + attributes.append(current.attr) + current = current.value + if not isinstance(current, ast.Name) or not attributes: + return None + return current.id, tuple(reversed(attributes)) + + +def _imported_bindings(statement: ast.stmt) -> tuple[str, ...]: + if isinstance(statement, ast.Import): + return tuple( + sorted( + alias.asname or alias.name.partition(".")[0] + for alias in statement.names + ) + ) + if isinstance(statement, ast.ImportFrom): + return tuple( + sorted( + alias.asname or alias.name + for alias in statement.names + if alias.name != "*" + ) + ) + return () + + +def import_bindings(statement: ast.stmt) -> tuple[ImportBinding, ...]: + """Retain qualified import targets without resolving calls during parsing.""" + + if isinstance(statement, ast.Import): + return _plain_import_bindings(statement) + if isinstance(statement, ast.ImportFrom) and statement.module is not None: + return _from_import_bindings(statement) + return () + + +def _plain_import_bindings(statement: ast.Import) -> tuple[ImportBinding, ...]: + bindings = {} + for alias in statement.names: + name = alias.asname or alias.name.partition(".")[0] + qualified = alias.name if alias.asname else alias.name.partition(".")[0] + bindings[name] = ImportBinding(name, qualified) + return tuple(sorted(bindings.values())) + + +def _from_import_bindings(statement: ast.ImportFrom) -> tuple[ImportBinding, ...]: + assert statement.module is not None + bindings = {} + for alias in statement.names: + if alias.name != "*": + name = alias.asname or alias.name + bindings[name] = ImportBinding(name, f"{statement.module}.{alias.name}") + return tuple(sorted(bindings.values())) + + +def _statement_restrictions(statement: ast.stmt) -> tuple[str, ...]: + restrictions = set() + if isinstance(statement, ast.ImportFrom) and statement.module == "__future__": + restrictions.add("future-import") + if isinstance(statement, ast.ImportFrom) and any( + alias.name == "*" for alias in statement.names + ): + restrictions.add("wildcard-import") + restricted_calls = {"eval", "exec", "globals", "locals"} + restrictions.update( + node.func.id + for node in ast.walk(statement) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in restricted_calls + ) + return tuple(sorted(restrictions)) + + +def _statement_action_root(statement: ast.stmt) -> ast.AST: + return statement.value if isinstance(statement, ast.Expr) else statement + + +def _component_span(component: _Component, source_index: _SourceIndex) -> SourceSpan: + if component.suite: + return source_index.covering_span(component.suite) + if isinstance(component.node, _SourceOccurrence): + return component.node.source_span + return source_index.span(component.node) + + +def _component_position(component: _Component) -> tuple[int, int]: + if isinstance(component.node, _SourceOccurrence): + return component.node.source_span.start + return _position(component.node) + + +type _FieldNode = ast.AST | _SourceOccurrence +_FIELD_NAME_MARKERS: dict[tuple[type[ast.AST], str], str] = { + (ast.ExceptHandler, "name"): "as", + (ast.FunctionDef, "name"): "def", + (ast.ClassDef, "name"): "class", +} +_FIELD_ITEM_ATTRIBUTES: dict[tuple[type[ast.AST], str], str] = { + (ast.ClassDef, "keywords"): "value", +} +_IMPLICIT_LOAD_FIELDS = frozenset({(ast.AugAssign, "target")}) +type _DirectFieldContext = tuple[ + _SourceIndex | None, + _CompoundProtocol | None, + Mapping[str, str], +] +type _NestedFieldContext = tuple[ + _SourceIndex | None, + str, + int, + _CompoundProtocol | None, + Mapping[str, str], +] + + +def _field_nodes( + node: ast.AST, + field_name: str, + value: object, + source_index: _SourceIndex | None, +) -> tuple[_FieldNode, ...]: + field_key = (type(node), field_name) + marker = _FIELD_NAME_MARKERS.get(field_key) + if marker is not None: + return _marked_field_node(node, marker, value, source_index) + item_attribute = _FIELD_ITEM_ATTRIBUTES.get(field_key) + if item_attribute is not None and isinstance(value, list): + return tuple( + projected + for item in value + if isinstance(item, ast.AST) + if isinstance((projected := getattr(item, item_attribute, None)), ast.AST) + ) + if isinstance(value, list): + return tuple(item for item in value if isinstance(item, ast.AST)) + return (value,) if isinstance(value, ast.AST) else () + + +def _marked_field_node( + node: ast.AST, + marker: str, + value: object, + source_index: _SourceIndex | None, +) -> tuple[_FieldNode, ...]: + if value is None: + return () + if not isinstance(value, str): + raise ForestBuildError("marked name field contains invalid data") + if source_index is None: + raise ForestBuildError("marked name field requires a source index") + return (source_index.marked_name(node, marker, value, ParameterRole.TARGET),) + + +def _components( + node: ast.AST, + source_index: _SourceIndex | None = None, +) -> tuple[_Component, ...]: + if isinstance(node, _ATOMIC_ACTIONS): + return () + compound_protocol = _COMPOUND_PROTOCOLS.get(type(node)) + aliases = _FIELD_NAMES.get(type(node), {}) + components: list[_Component] = [] + for field_name, value in ast.iter_fields(node): + components.extend( + _direct_field_components( + node, + field_name, + value, + (source_index, compound_protocol, aliases), + ) + ) + return tuple(sorted(components, key=_component_position)) + + +def _direct_field_components( + node: ast.AST, + field_name: str, + value: object, + context: _DirectFieldContext, +) -> list[_Component]: + source_index, protocol, aliases = context + if field_name in {"ctx", "ops", "op", "type_comment", "simple", "level"}: + return [] + if (type(node), field_name) in _UNSUPPORTED_NONEMPTY_FIELDS and value: + raise ForestBuildError( + f"unsupported non-empty field: {type(node).__name__}.{field_name}" + ) + if protocol is not None and field_name not in protocol.fields: + return [] + if protocol is not None and field_name in protocol.suites: + statements = tuple(item for item in value if isinstance(item, ast.stmt)) + return ( + [ + _Component( + aliases.get(field_name, field_name), + statements[0], + field_name, + 0, + suite=statements, + ) + ] + if statements + else [] + ) + return _direct_field_items(node, field_name, value, source_index, aliases) + + +def _direct_field_items( + node: ast.AST, + field_name: str, + value: object, + source_index: _SourceIndex | None, + aliases: Mapping[str, str], +) -> list[_Component]: + components = [] + field_nodes = _field_nodes(node, field_name, value, source_index) + for item_index, item in enumerate(field_nodes): + base_name = aliases.get(field_name, field_name) + component_name = _component_name(node, field_name, base_name, item_index, value) + if isinstance(item, _SourceOccurrence) or ( + type(item) not in _STRUCTURE_PROTOCOLS and _has_span(item) + ): + components.append(_Component(component_name, item, field_name, item_index)) + else: + components.extend( + _span_components( + item, + prefix=component_name, + source_index=source_index, + origin_field=field_name, + origin_index=item_index, + ) + ) + return components + + +def _component_name( + node: ast.AST, + field_name: str, + base_name: str, + item_index: int, + field_value: object, +) -> str: + is_list = isinstance(field_value, list) + item_count = len(field_value) if is_list else 1 + if isinstance(node, ast.BoolOp) and field_name == "values" and item_count == 2: + return ("left", "right")[item_index] + if isinstance(node, ast.Call) and field_name == "args": + return f"positional_{item_index}" + if isinstance(node, ast.Call) and field_name == "keywords": + return f"keyword_{item_index}" + always_indexed = isinstance( + node, + (ast.List, ast.Tuple, ast.Set, ast.Dict, ast.Compare), + ) + if is_list and (item_count > 1 or always_indexed): + return f"{base_name}_{item_index}" + return base_name + + +def _span_components( + node: ast.AST, + *, + prefix: str, + source_index: _SourceIndex | None, + origin_field: str, + origin_index: int, +) -> list[_Component]: + node_protocol = _STRUCTURE_PROTOCOLS.get(type(node)) + aliases = _FIELD_NAMES.get(type(node), {}) + found: list[_Component] = [] + for field_name, value in ast.iter_fields(node): + found.extend( + _nested_field_components( + node, + field_name, + value, + prefix, + (source_index, origin_field, origin_index, node_protocol, aliases), + ) + ) + return found + + +def _nested_field_components( + node: ast.AST, + field_name: str, + value: object, + prefix: str, + context: _NestedFieldContext, +) -> list[_Component]: + source_index, origin_field, origin_index, protocol, aliases = context + if field_name in {"ctx", "ops", "op", "type_comment"}: + return [] + if protocol is not None and field_name not in protocol.fields: + return [] + if protocol is not None and field_name in protocol.suites: + statements = tuple(item for item in value if isinstance(item, ast.stmt)) + return ( + [ + _Component( + f"{prefix}_{aliases.get(field_name, field_name)}", + statements[0], + origin_field, + origin_index, + suite=statements, + ) + ] + if statements + else [] + ) + return _nested_field_items(node, field_name, value, prefix, context) + + +def _nested_field_items( + node: ast.AST, + field_name: str, + value: object, + prefix: str, + context: _NestedFieldContext, +) -> list[_Component]: + source_index, origin_field, origin_index, _protocol, aliases = context + found = [] + field_nodes = _field_nodes(node, field_name, value, source_index) + for item_index, item in enumerate(field_nodes): + base_name = aliases.get(field_name, field_name) + suffix = f"{base_name}_{item_index}" if isinstance(value, list) else base_name + name = f"{prefix}_{suffix}" + if isinstance(item, _SourceOccurrence) or ( + type(item) not in _STRUCTURE_PROTOCOLS and _has_span(item) + ): + found.append(_Component(name, item, origin_field, origin_index)) + else: + found.extend( + _span_components( + item, + prefix=name, + source_index=source_index, + origin_field=origin_field, + origin_index=origin_index, + ) + ) + return found + + +def _parameter_action_roots(node: _FieldNode) -> tuple[ast.AST, ...]: + if isinstance(node, _SourceOccurrence): + return () + if isinstance(node, ast.expr): + if not isinstance(getattr(node, "ctx", ast.Load()), (ast.Store, ast.Del)): + return (node,) + roots: list[ast.AST] = [] + for component in _components(node): + roots.extend(_parameter_action_roots(component.node)) + return tuple(_outermost_nonoverlapping(roots)) + + +def _parameter_action_children( + component: _Component, owner: ast.AST +) -> tuple[ast.AST, ...]: + roots = component.child_nodes + if not _component_has_implicit_load(component, owner): + return roots + implicit = tuple( + ast.copy_location(ast.Name(id=item.id, ctx=ast.Load()), item) + for item in ast.walk(component.node) + if isinstance(item, ast.Name) and isinstance(item.ctx, ast.Store) + ) + return tuple(_outermost_nonoverlapping((*roots, *implicit))) + + +def _component_has_implicit_load(component: _Component, owner: ast.AST) -> bool: + return ( + isinstance(component.node, ast.AST) + and (type(owner), component.origin_field) in _IMPLICIT_LOAD_FIELDS + ) + + +def _parameter_role(node: ast.AST) -> ParameterRole: + """Classify a source hole from Python's own expression context.""" + + if isinstance(node, ast.pattern): + return ParameterRole.TARGET + context = getattr(node, "ctx", None) + if isinstance(context, (ast.Store, ast.Del)): + return ParameterRole.TARGET + return ParameterRole.VALUE + + +def _outermost_nonoverlapping(nodes: Iterable[ast.AST]) -> list[ast.AST]: + ordered = sorted(nodes, key=lambda item: (_position(item), -_span_size(item))) + selected: list[ast.AST] = [] + for node in ordered: + if any(_contains(existing, node) for existing in selected): + continue + selected.append(node) + return selected + + +def _unique_component_name(components: tuple[_Component, ...], index: int) -> str: + name = components[index].name + duplicates = [item for item in components if item.name == name] + if len(duplicates) == 1: + return name + occurrence = sum(1 for item in components[:index] if item.name == name) + return f"{name}_{occurrence}" + + +def _sentinel(name: str) -> str: + """The written stand-in for one removed child. + + This helper owns the sentinel's shape, and it is write-only: a + hole is found by its recorded position, never by searching for this text. + Its whole purpose is to keep a template readable, and to leave it + byte-identical to its source when that source contains no braces. + """ + + return "{" + name + "}" + + +def _replace_source( + source_text: str, + container: SourceSpan, + replacements: Iterable[_ChildCut], +) -> TemplateSource: + """Cut every child span out of its owner, recording where each hole sits. + + The template is assembled left to right, so each recorded position is + already a position in the finished template. Each child's name arrives with + its span rather than being read back out of the sentinel: recovering a name + by slicing the written text would leave the sentinel's shape load-bearing, + which is the coupling positional holes exist to remove. + """ + + ordered = sorted(replacements, key=lambda item: item[0].start_offset) + pieces: list[str] = [] + holes: list[TemplateHole] = [] + origins = _OriginTrail(container.start_line) + written = 0 + cursor = container.start_offset + for span, name in ordered: + _require_contained(span, container, cursor) + literal = source_text[ + cursor - container.start_offset : span.start_offset - container.start_offset + ] + written += len(literal) + sentinel = _sentinel(name) + pieces.extend((literal, sentinel)) + holes.append(TemplateHole(written, written + len(sentinel), name)) + origins.advance(literal, span.end_line - span.start_line) + written += len(sentinel) + cursor = span.end_offset + trailing = source_text[cursor - container.start_offset :] + pieces.append(trailing) + origins.advance(trailing, 0) + return TemplateSource("".join(pieces), tuple(holes), origins.recorded()) + + +class _OriginTrail: + """Track which source line each template line begins on, as one is built. + + A literal stretch copied from the owner advances the source line once per + newline it carries, and each of those newlines also starts a template line. + A removed child advances the source line by its own height while starting no + template line at all, because its sentinel is one line. Keeping both in one + place is what stops the two from being derived from each other later. + """ + + def __init__(self, first_line: int) -> None: + self._source_line = first_line + self._lines = [first_line] + + def advance(self, literal: str, child_height: int) -> None: + """Consume one copied stretch, then the child that was cut out after it.""" + + for _newline in range(literal.count("\n")): + self._source_line += 1 + self._lines.append(self._source_line) + self._source_line += child_height + + def recorded(self) -> SourceLineNumbers: + """The source line every template line began on, in template order.""" + + return tuple(self._lines) + + +def _require_contained(span: SourceSpan, container: SourceSpan, cursor: int) -> None: + """A child must lie inside its owner and after every earlier sibling.""" + + if ( + span.start_offset < container.start_offset + or span.end_offset > container.end_offset + ): + raise ForestBuildError("child source span escapes its owner") + if span.start_offset < cursor: + raise ForestBuildError("overlapping source spans cannot form Parameters") + + +def _fill(template: str, holes: TemplateHoles, values: dict[str, str]) -> str: + """Splice each child's source into its recorded hole, right to left. + + Filling by position rather than by pattern is what makes exact + reconstruction sound: text a child contributes is never re-scanned, so a + child whose own source contains `{0}` cannot capture a later fill, and no + ordering heuristic is needed. + """ + + if not holes: + return template + result = template + for hole in sorted(holes, reverse=True): + result = result[: hole.start] + values[hole.name] + result[hole.end :] + return result + + +def _validate_placeholders( + holes: TemplateHoles, + expected: set[str], +) -> None: + """Every child owns exactly one hole, and no hole is unclaimed.""" + + declared = [hole.name for hole in holes] + if sorted(declared) != sorted(expected): + raise ForestBuildError("template does not declare every child") + + +def _has_span(node: ast.AST) -> bool: + return all( + getattr(node, field, None) is not None + for field in ("lineno", "col_offset", "end_lineno", "end_col_offset") + ) + + +def _span_is_inside(inner: SourceSpan, outer: SourceSpan) -> bool: + return ( + inner.start_offset >= outer.start_offset + and inner.end_offset <= outer.end_offset + ) + + +def _position(node: ast.AST) -> tuple[int, int, int, int]: + return ( + getattr(node, "lineno", -1), + getattr(node, "col_offset", -1), + getattr(node, "end_lineno", -1), + getattr(node, "end_col_offset", -1), + ) + + +def _span_size(node: ast.AST) -> int: + return ( + (getattr(node, "end_lineno", 0) - getattr(node, "lineno", 0)) * 1_000_000 + + getattr(node, "end_col_offset", 0) + - getattr(node, "col_offset", 0) + ) + + +def _contains(outer: ast.AST, inner: ast.AST) -> bool: + return ( + _position(outer)[:2] <= _position(inner)[:2] + and _position(outer)[2:] >= _position(inner)[2:] + ) diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/source.py b/py2udf/src/main/python/python_to_workflow/mosaic/source.py new file mode 100644 index 00000000000..3869242a30e --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/source.py @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Single-parse source boundary for the modular compiler.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from python_to_workflow.mosaic.forest import ( + AliasValueFlow, + BindingValueFlow, + BuiltSource, + CallInput, + CallInputs, + ContentLoadFlow, + DynamicCall, + FreshValueFlow, + ImportBinding, + ImportOccurrence, + MethodCallFlow, + MethodEffect, + NamedCall, + ReceiverCall, + SourceCall, + SourceInventory, + SourceStatement, + UnknownValueFlow, + build_forest_with_inventory, +) + + +@dataclass(frozen=True) +class GeneratedSymbols: + """Fresh Python identifiers owned by one generated workflow program.""" + + workflow_module: str + runtime: str + heap: str + boundary: str + driver: str + operator: str + + def __post_init__(self) -> None: + names = ( + self.workflow_module, + self.runtime, + self.heap, + self.boundary, + self.driver, + self.operator, + ) + if any(not name.isidentifier() for name in names): + raise ValueError("generated symbols must be Python identifiers") + if len(set(names)) != len(names): + raise ValueError("generated symbols must be distinct") + + +@dataclass(frozen=True) +class ParsedSource: + """One source parse plus the generated names fresh for that source.""" + + built: BuiltSource + symbols: GeneratedSymbols + + @property + def forest(self): + """Return the source-exact ActionForest from the parsed source.""" + + return self.built.forest + + @property + def inventory(self) -> SourceInventory: + """Return the source inventory from the parsed source.""" + + return self.built.inventory + + def statement(self, root_id: str) -> SourceStatement: + """Resolve one top-level statement through the source-owned index.""" + + return self.built.statement(root_id) + + +def parse_source(source: str, /) -> ParsedSource: + """Build the source-exact forest and admission inventory in one parse.""" + + built = build_forest_with_inventory(source) + return ParsedSource(built, _generated_symbols(built.inventory)) + + +def _generated_symbols(inventory: SourceInventory) -> GeneratedSymbols: + """Allocate deterministic generated identifiers outside the source namespace.""" + + reserved = { + name + for statement in inventory.statements + for name in ( + *statement.reads, + *statement.writes, + *statement.in_place_writes, + *statement.mutates, + *statement.imports, + *(binding.name for binding in statement.import_bindings), + ) + } + allocated: set[str] = set() + + def fresh(role: str) -> str: + index = 0 + while True: + candidate = f"_mosaic_{role}_{index}" + if candidate not in reserved and candidate not in allocated: + allocated.add(candidate) + return candidate + index += 1 + + return GeneratedSymbols( + fresh("workflow_module"), + fresh("runtime"), + fresh("heap"), + fresh("boundary"), + fresh("driver"), + fresh("operator"), + ) + + +__all__ = [ + "AliasValueFlow", + "BindingValueFlow", + "CallInput", + "CallInputs", + "ContentLoadFlow", + "DynamicCall", + "FreshValueFlow", + "ImportBinding", + "ImportOccurrence", + "MethodCallFlow", + "MethodEffect", + "NamedCall", + "GeneratedSymbols", + "ParsedSource", + "SourceInventory", + "ReceiverCall", + "SourceCall", + "SourceStatement", + "UnknownValueFlow", + "parse_source", +] diff --git a/py2udf/src/test/python/python_to_workflow/mosaic/test_forest.py b/py2udf/src/test/python/python_to_workflow/mosaic/test_forest.py new file mode 100644 index 00000000000..3417c453c99 --- /dev/null +++ b/py2udf/src/test/python/python_to_workflow/mosaic/test_forest.py @@ -0,0 +1,259 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import ast + +import pytest +from python_to_workflow.mosaic.forest import NameAccess +from python_to_workflow.mosaic.source import ( + DynamicCall, + ImportBinding, + ImportOccurrence, + NamedCall, + ReceiverCall, + parse_source, +) + + +def test_source_builds_forest_and_inventory_from_one_parse( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = 0 + original = ast.parse + + def counted_parse(*args: object, **kwargs: object) -> ast.Module: + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(ast, "parse", counted_parse) + + parsed = parse_source("left = 1\nright = left + 1\n") + parsed.forest.validate() + + assert calls == 1 + assert len(parsed.forest.roots) == len(parsed.inventory.statements) == 2 + + +def test_module_occurrences_exclude_comprehension_locals_but_keep_first_iter() -> None: + parsed = parse_source("values = [x for x in x]\n") + + occurrences = parsed.forest.module_name_occurrences( + parsed.forest.roots[0], + frozenset({"x"}), + ) + + assert tuple(row.access for row in occurrences) == (NameAccess.LOAD,) + assert tuple( + parsed.forest.source[row.span.start_offset : row.span.end_offset] + for row in occurrences + ) == ("x",) + + +def test_forest_reports_only_module_activation_binders() -> None: + parsed = parse_source( + "try:\n" + " pass\n" + "except RuntimeError as error:\n" + " pass\n" + "match value:\n" + " case {'item': captured, **rest}:\n" + " pass\n" + ) + + first, second = parsed.forest.roots + + assert parsed.forest.python_activation_bindings(first) == frozenset({"error"}) + assert parsed.forest.python_activation_bindings(second) == frozenset( + {"captured", "rest"} + ) + + +def test_generated_scaffold_symbols_are_all_fresh_from_the_source_inventory() -> None: + """No composer or executor identifier may live outside symbol authority.""" + + source = "\n".join( + f"_mosaic_{role}_0 = {index}" + for index, role in enumerate( + ( + "workflow_module", + "runtime", + "heap", + "boundary", + "driver", + "operator", + ) + ) + ) + + symbols = parse_source(source + "\n").symbols + + assert ( + symbols.workflow_module, + symbols.runtime, + symbols.heap, + symbols.boundary, + symbols.driver, + symbols.operator, + ) == tuple( + f"_mosaic_{role}_1" + for role in ( + "workflow_module", + "runtime", + "heap", + "boundary", + "driver", + "operator", + ) + ) + + +def test_inventory_exposes_closed_call_shapes_from_the_same_parse() -> None: + """Call evidence is typed analysis input, never renderer inference.""" + + parsed = parse_source( + "print(value)\nitems.copy()\nregistry['runner'](value, mode=option)\n" + ) + + direct, receiver, dynamic = ( + statement.calls[0] for statement in parsed.inventory.statements + ) + + assert isinstance(direct, NamedCall) + assert direct.action_id == "s1" + assert direct.name == "print" + assert direct.inputs.bindings() == ("value",) + assert isinstance(receiver, ReceiverCall) + assert receiver.action_id == "s2" + assert (receiver.receiver, receiver.method) == ("items", "copy") + assert isinstance(dynamic, DynamicCall) + assert dynamic.action_id == "s3" + assert dynamic.callable.bindings == ("registry",) + assert dynamic.inputs.bindings() == ("option", "value") + + +def test_inventory_retains_import_targets_and_attribute_call_paths() -> None: + """Effect resolution uses typed import and call facts from parsing.""" + + parsed = parse_source( + "import random as rng\nimport numpy as np\nrng.seed(7)\nnp.random.seed(8)\n" + ) + + first, second, seed, numpy_seed = parsed.inventory.statements + assert first.import_bindings == (ImportBinding("rng", "random"),) + assert second.import_bindings == (ImportBinding("np", "numpy"),) + assert isinstance(seed.calls[0], ReceiverCall) + assert seed.calls[0].attributes == ("seed",) + assert isinstance(numpy_seed.calls[0], ReceiverCall) + assert numpy_seed.calls[0].attributes == ("random", "seed") + + +def test_forest_indexes_outer_roots_once_for_all_consumers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Repeated root lookups consume the forest index, not ancestry walks.""" + + forest = parse_source( + "if condition:\n" + " result = transform(source)\n" + "else:\n" + " result = fallback(source)\n" + ).forest + expected = {action.id: forest.root_of(action.id) for action in forest.actions} + + def unexpected_walk(_forest: object, _action_id: str) -> str | None: + raise AssertionError("root lookup repeated the structural ancestry walk") + + monkeypatch.setattr(type(forest), "parent_action", unexpected_walk) + + actual = {action.id: forest.root_of(action.id) for action in forest.actions} + assert actual == expected + + +def test_parsed_source_indexes_statement_summaries_once() -> None: + """Realizations resolve source facts without rescanning all statements.""" + + parsed = parse_source("left = 1\nright = left + 1\n") + expected = tuple(parsed.statement(root) for root in parsed.forest.roots) + + class _NoIteration(tuple): + def __iter__(self): + raise AssertionError("statement lookup rescanned the inventory") + + object.__setattr__( + parsed.inventory, + "statements", + _NoIteration(parsed.inventory.statements), + ) + + assert tuple(parsed.statement(root) for root in parsed.forest.roots) == expected + + +def test_inventory_owns_import_occurrences_for_nested_and_from_imports() -> None: + """Resolution consumes typed per-Action imports, never forest internals.""" + + parsed = parse_source( + "if condition:\n import random as rng\nfrom numpy import random as nr\n" + ) + + assert parsed.inventory.import_occurrences == ( + ImportOccurrence("s1.body.0", (ImportBinding("rng", "random"),)), + ImportOccurrence("s2", (ImportBinding("nr", "numpy.random"),)), + ) + + +@pytest.mark.parametrize( + ("source", "expected"), + ( + ( + "import random as rng, numpy.random as rng\n", + ImportBinding("rng", "numpy.random"), + ), + ( + "import numpy.random as rng, random as rng\n", + ImportBinding("rng", "random"), + ), + ), +) +def test_duplicate_import_aliases_preserve_python_last_wins( + source: str, + expected: ImportBinding, +) -> None: + """Canonicalization happens after source-order binding semantics.""" + + parsed = parse_source(source) + + assert parsed.inventory.statements[0].import_bindings == (expected,) + assert parsed.inventory.import_occurrences == (ImportOccurrence("s1", (expected,)),) + + +def test_lambda_body_calls_are_dormant_but_default_calls_execute() -> None: + """Call facts follow Python definition-time execution, not raw AST walk.""" + + parsed = parse_source("callback = lambda value=seed(): hidden(value)\n") + calls = parsed.inventory.statements[0].calls + + assert len(calls) == 1 + assert isinstance(calls[0], NamedCall) + assert calls[0].name == "seed" + + +def test_parse_source_rejects_malformed_python() -> None: + with pytest.raises(SyntaxError): + parse_source("value =\n") From 912e15522fba833f658c12bc60f4398a7410b22d Mon Sep 17 00:00:00 2001 From: carloea2 Date: Tue, 1 Sep 2026 17:13:35 -0600 Subject: [PATCH 2/3] feat(py2udf): add dependency graph --- .../python/python_to_workflow/mosaic/graph.py | 130 +++++ .../python/python_to_workflow/mosaic/model.py | 454 ++++++++++++++++++ .../python_to_workflow/mosaic/test_graph.py | 97 ++++ 3 files changed, 681 insertions(+) create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/graph.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/model.py create mode 100644 py2udf/src/test/python/python_to_workflow/mosaic/test_graph.py diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/graph.py b/py2udf/src/main/python/python_to_workflow/mosaic/graph.py new file mode 100644 index 00000000000..ad9bc355339 --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/graph.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Contextual dependency graph.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from python_to_workflow.mosaic.model import ( + ActionTransition, + Carrier, + ContextualActionId, + ReachingPair, + reaching_pair_key, +) + + +class InvalidDependenceGraphError(ValueError): + """Graph rows are noncanonical or contradict their endpoint transitions.""" + + +@dataclass(frozen=True) +class ActionDependenceGraph: + """Action dependencies independent of source rendering.""" + + transitions: tuple[ActionTransition[ContextualActionId], ...] + reaching_pairs: tuple[ReachingPair[ContextualActionId], ...] + declared_inputs: frozenset[Carrier] + + def __post_init__(self) -> None: + transition_ids = tuple(row.action_id for row in self.transitions) + canonical_pairs = tuple(sorted(set(self.reaching_pairs), key=_pair_key)) + if ( + any( + not isinstance(action_id, ContextualActionId) + for action_id in transition_ids + ) + or len(transition_ids) != len(set(transition_ids)) + or self.transitions + != tuple(sorted(self.transitions, key=lambda row: row.action_id)) + or self.reaching_pairs != canonical_pairs + ): + raise InvalidDependenceGraphError( + "dependence graph must contain canonical contextual Action rows" + ) + _validate_pair_incidence(self.transitions, self.reaching_pairs) + _validate_requirement_supply( + self.transitions, + self.reaching_pairs, + self.declared_inputs, + ) + _validate_declared_inputs(self.transitions, self.declared_inputs) + + +def _validate_pair_incidence( + transitions: tuple[ActionTransition[ContextualActionId], ...], + pairs: tuple[ReachingPair[ContextualActionId], ...], +) -> None: + by_action = {transition.action_id: transition for transition in transitions} + for pair in pairs: + if ( + pair.producer_action not in by_action + or pair.consumer_action not in by_action + or pair.carrier not in by_action[pair.producer_action].establishes + or pair.carrier not in by_action[pair.consumer_action].requires + ): + raise InvalidDependenceGraphError( + "dependence pair is incompatible with its endpoint transitions" + ) + + +def _validate_requirement_supply( + transitions: tuple[ActionTransition[ContextualActionId], ...], + pairs: tuple[ReachingPair[ContextualActionId], ...], + declared_inputs: frozenset[Carrier], +) -> None: + """Require every contextual input to have one graph or entry supply.""" + + reached = {(pair.consumer_action, pair.carrier) for pair in pairs} + orphan = next( + ( + (transition.action_id, carrier) + for transition in transitions + for carrier in transition.requires + if carrier not in declared_inputs + and (transition.action_id, carrier) not in reached + ), + None, + ) + if orphan is not None: + raise InvalidDependenceGraphError( + f"dependence requirement has no supply: {orphan!r}" + ) + + +def _validate_declared_inputs( + transitions: tuple[ActionTransition[ContextualActionId], ...], + declared_inputs: frozenset[Carrier], +) -> None: + """Reject entry declarations that no exact Action actually requires.""" + + required = { + carrier for transition in transitions for carrier in transition.requires + } + unused = declared_inputs - required + if unused: + raise InvalidDependenceGraphError( + f"declared input has no consumer: {sorted(map(repr, unused))!r}" + ) + + +def _pair_key( + pair: ReachingPair[ContextualActionId], +) -> tuple[object, ...]: + return reaching_pair_key(pair) diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/model.py b/py2udf/src/main/python/python_to_workflow/mosaic/model.py new file mode 100644 index 00000000000..2c968fbea87 --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/model.py @@ -0,0 +1,454 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Named immutable records shared by analysis, checking, and realization.""" + +from __future__ import annotations + +from collections.abc import Hashable +from dataclasses import dataclass, field, fields, is_dataclass +from enum import Enum +from functools import total_ordering +from typing import Protocol, TypeVar + +from python_to_workflow.mosaic.forest import ActionForest + +ActionIdentity = TypeVar("ActionIdentity", bound=Hashable) +type LexicalScopeId = Hashable + + +@total_ordering +@dataclass(frozen=True) +class ContextualActionId: + """One source Action in one static context and lexical scope.""" + + source_action_id: str + context_id: str + scope_id: LexicalScopeId + + def __post_init__(self) -> None: + if not self.source_action_id or not self.context_id: + raise ValueError("contextual Action identity components must be nonempty") + try: + hash(self.scope_id) + except TypeError as error: + raise TypeError("lexical scope identity must be hashable") from error + + def __lt__(self, other: object) -> bool: + if not isinstance(other, ContextualActionId): + return NotImplemented + return ( + self.source_action_id, + self.context_id, + repr(self.scope_id), + ) < ( + other.source_action_id, + other.context_id, + repr(other.scope_id), + ) + + +@dataclass(frozen=True, eq=False) +class Carrier: + """A semantic value with structural identity.""" + + identity: Hashable + _key: CarrierIdentityKey = field(init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_key", carrier_identity_key(self.identity)) + + def __hash__(self) -> int: + return hash(self._key) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Carrier): + return NotImplemented + return self._key == other._key + + @property + def identity_key(self) -> CarrierIdentityKey: + """Return the canonical identity computed once at construction.""" + + return self._key + + +@dataclass(frozen=True, order=True) +class CarrierIdentityKey: + """Structural identity used for equality and ordering.""" + + encoded: tuple[object, ...] + + +def carrier_identity_key(identity: Hashable, /) -> CarrierIdentityKey: + """Return a deterministic key for supported immutable identity values.""" + + try: + hash(identity) + except RecursionError as error: + raise TypeError("Carrier identity contains a cyclic structure") from error + except TypeError as error: + raise TypeError("Carrier identity must be hashable") from error + try: + return CarrierIdentityKey(_identity_node(identity, frozenset())) + except RecursionError as error: + raise TypeError("Carrier identity contains a cyclic structure") from error + + +def _identity_node(value: object, active: frozenset[int]) -> tuple[object, ...]: + """Encode typed immutable structure without consulting ``repr``.""" + + _reject_identity_cycle(value, active) + return _uncycled_identity_node(value, active | {id(value)}) + + +def _reject_identity_cycle(value: object, active: frozenset[int]) -> None: + """Fail closed when one structural identity revisits an active object.""" + + if id(value) in active: + raise TypeError("Carrier identity contains a cyclic structure") + + +def _uncycled_identity_node( + value: object, active: frozenset[int] +) -> tuple[object, ...]: + """Encode one value after the active-path cycle check.""" + + enum = _enum_identity_node(value) + if enum is not None: + return enum + primitive = _primitive_identity_node(value) + if primitive is not None: + return primitive + collection = _collection_identity_node(value, active) + if collection is not None: + return collection + record = _record_identity_node(value, active) + if record is not None: + return record + raise TypeError( + "Carrier identity must be an immutable primitive, tuple, frozenset, " + "dataclass, Enum, or define __mosaic_identity_key__()" + ) + + +def _enum_identity_node(value: object) -> tuple[object, ...] | None: + """Encode nominal enum identity before scalar subclass coercions.""" + + if isinstance(value, Enum): + return ("enum", _identity_type_name(value), value.name) + return None + + +def _primitive_identity_node(value: object) -> tuple[object, ...] | None: + """Encode one supported scalar identity or decline structural values.""" + + if value is None: + return ("none",) + if isinstance(value, bool): + return ("bool", value) + numeric = _numeric_identity_node(value) + if numeric is not None: + return numeric + return _text_identity_node(value) + + +def _numeric_identity_node(value: object) -> tuple[object, ...] | None: + """Encode a non-boolean numeric identity with exact representation.""" + + if isinstance(value, int): + return ("int", str(value)) + if isinstance(value, float): + return ("float", value.hex()) + return None + + +def _text_identity_node(value: object) -> tuple[object, ...] | None: + """Encode textual and binary scalar identities.""" + + if isinstance(value, str): + return ("str", value) + if isinstance(value, bytes): + return ("bytes", value.hex()) + return None + + +def _collection_identity_node( + value: object, active: frozenset[int] +) -> tuple[object, ...] | None: + """Encode supported ordered and unordered immutable collections.""" + + if isinstance(value, tuple): + return ("tuple", *(_identity_node(item, active) for item in value)) + if isinstance(value, frozenset): + return ("frozenset", *_sorted_identity_nodes(value, active)) + return None + + +def _record_identity_node( + value: object, active: frozenset[int] +) -> tuple[object, ...] | None: + """Encode dataclass or explicit extension identities.""" + + type_name = _identity_type_name(value) + if is_dataclass(value) and not isinstance(value, type): + return ( + "dataclass", + type_name, + *( + (field.name, _identity_node(getattr(value, field.name), active)) + for field in fields(value) + ), + ) + custom = getattr(value, "__mosaic_identity_key__", None) + if callable(custom): + return ("custom", type_name, _identity_node(custom(), active)) + return None + + +def _identity_type_name(value: object) -> str: + """Return the stable qualified type tag for one structural identity.""" + + return f"{type(value).__module__}.{type(value).__qualname__}" + + +def _sorted_identity_nodes( + values: frozenset[object], active: frozenset[int] +) -> tuple[tuple[object, ...], ...]: + """Canonicalize an unordered identity collection without using repr.""" + + return tuple(sorted(_identity_node(item, active) for item in values)) + + +@dataclass(frozen=True, order=True) +class ModuleBinding: + """One Python name in the source module's logical namespace.""" + + name: str + + def __post_init__(self) -> None: + if not self.name.isidentifier(): + raise ValueError("module binding name must be a Python identifier") + + +@dataclass(frozen=True, order=True) +class PythonActivationBinding: + """A syntax-owned binding whose lifetime stays in one Python activation.""" + + name: str + binder_action_id: str + + def __post_init__(self) -> None: + if not self.name.isidentifier(): + raise ValueError("activation binding name must be a Python identifier") + if not self.binder_action_id: + raise ValueError("activation binder Action id must be nonempty") + + +@dataclass(frozen=True, order=True) +class EffectState: + """One ambient state domain whose continuity requires a realization.""" + + domain: str + + def __post_init__(self) -> None: + if not self.domain or not self.domain.replace("-", "_").isidentifier(): + raise ValueError("effect-state domain must be a nonempty identifier") + + +@dataclass(frozen=True, order=True) +class AmbientModuleState: + """Ambient state owned by one exact imported module path.""" + + module_path: tuple[str, ...] + + def __post_init__(self) -> None: + if not self.module_path or any( + not component.isidentifier() for component in self.module_path + ): + raise ValueError("ambient module path must contain Python identifiers") + + +@dataclass(frozen=True) +class ProcessAmbientState: + """Unknown process state reachable through an unresolved call alternative.""" + + +@dataclass(frozen=True) +class WorldOrder: + """Conservative sequencing token removed only by stronger semantics.""" + + +@dataclass(frozen=True) +class ReachingPair[ActionIdentity: Hashable]: + """Exact dependence from one producing Action to one consumer.""" + + producer_action: ActionIdentity + consumer_action: ActionIdentity + carrier: Carrier + + +def reaching_pair_key[PairAction: Hashable]( + pair: ReachingPair[PairAction], / +) -> tuple[PairAction, PairAction, CarrierIdentityKey]: + """Return the one canonical order key for semantic pair evidence.""" + + return pair.producer_action, pair.consumer_action, pair.carrier.identity_key + + +@dataclass(frozen=True) +class CarrierIncidence: + """One Carrier and its exact incident contextual dependence pairs.""" + + carrier: Carrier + pairs: tuple[ReachingPair[ContextualActionId], ...] + + +@dataclass(frozen=True) +class RealizationBatch: + """An Action cover and its distinct Carrier incidences.""" + + actions: tuple[ContextualActionId, ...] + incidences: tuple[CarrierIncidence, ...] + + +@dataclass(frozen=True) +class ActionTransition[ActionIdentity: Hashable]: + """Carriers required and potentially established by one Action.""" + + action_id: ActionIdentity + requires: frozenset[Carrier] + establishes: frozenset[Carrier] + + +@dataclass(frozen=True) +class EntryDemand: + """One external Carrier required by one exact contextual Action.""" + + consumer_action: ContextualActionId + carrier: Carrier + + +@dataclass(frozen=True) +class ActionProjection: + """Generated Action forest plus exact semantic coverage receipts.""" + + fragment: ActionForest + covers: frozenset[ContextualActionId] + materializes: frozenset[Carrier] + + +@dataclass(frozen=True) +class InternalProjection: + """One certified local program.""" + + program: ActionProjection + + +@dataclass(frozen=True) +class BoundaryProjection: + """Certified programs on the export and import sides of a boundary.""" + + export: ActionProjection + import_: ActionProjection + + +@dataclass(frozen=True) +class UDFActionForest: + """One complete generated target-runtime module.""" + + forest: ActionForest + + +@dataclass(frozen=True) +class WireForm: + """Exactly the materialized fields of one boundary realization.""" + + fields: tuple[str, ...] + + +class InternalApplicability(Protocol): + """Tamper guard for one prepared internal realization application.""" + + def __call__(self, batch: RealizationBatch, /) -> bool: ... + + +class BoundaryApplicability(Protocol): + """Tamper guard for one prepared boundary realization application.""" + + def __call__(self, batch: RealizationBatch, /) -> bool: ... + + +class InternalRenderer(Protocol): + """Render one certified local batch against its source forest.""" + + def __call__( + self, forest: ActionForest, batch: RealizationBatch, / + ) -> InternalProjection: ... + + +class BoundaryRenderer(Protocol): + """Render one certified crossing under its exact selected WireForm.""" + + def __call__( + self, batch: RealizationBatch, wire: WireForm, / + ) -> BoundaryProjection: ... + + +@dataclass(frozen=True) +class InternalRealization: + """One legal local projection family for an exact Action batch.""" + + realization_id: str + applicability: InternalApplicability + realize: InternalRenderer + + +@dataclass(frozen=True) +class BoundaryRealization: + """One legal crossing family with inert wire and executable projection.""" + + realization_id: str + applicability: BoundaryApplicability + wire: WireForm + realize: BoundaryRenderer + + +@dataclass(frozen=True) +class InternalApplication: + """One provider-certified local realization for one exact batch.""" + + realization: InternalRealization + batch: RealizationBatch + cost: int + + def __post_init__(self) -> None: + if not isinstance(self.cost, int) or self.cost < 0: + raise ValueError("internal application cost must be nonnegative") + + +@dataclass(frozen=True) +class BoundaryApplication: + """One provider-certified crossing realization for one exact batch.""" + + realization: BoundaryRealization + batch: RealizationBatch + cost: int + + def __post_init__(self) -> None: + if not isinstance(self.cost, int) or self.cost < 0: + raise ValueError("boundary application cost must be nonnegative") diff --git a/py2udf/src/test/python/python_to_workflow/mosaic/test_graph.py b/py2udf/src/test/python/python_to_workflow/mosaic/test_graph.py new file mode 100644 index 00000000000..21bd075e8b4 --- /dev/null +++ b/py2udf/src/test/python/python_to_workflow/mosaic/test_graph.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pytest +from python_to_workflow.mosaic.graph import ( + ActionDependenceGraph, + InvalidDependenceGraphError, +) +from python_to_workflow.mosaic.model import ( + ActionTransition, + Carrier, + ContextualActionId, + ReachingPair, +) + + +def _graph_rows() -> tuple[ + tuple[ActionTransition[ContextualActionId], ...], + tuple[ReachingPair[ContextualActionId], ...], +]: + producer = ContextualActionId("s1", "module", "module") + consumer = ContextualActionId("s2", "module", "module") + carrier = Carrier("value") + pair = ReachingPair(producer, consumer, carrier) + transitions = ( + ActionTransition(producer, frozenset(), frozenset({carrier})), + ActionTransition(consumer, frozenset({carrier}), frozenset()), + ) + return transitions, (pair,) + + +def test_dependence_graph_accepts_exact_endpoint_incidence() -> None: + transitions, pairs = _graph_rows() + + graph = ActionDependenceGraph(transitions, pairs, frozenset()) + + assert graph.reaching_pairs == pairs + + +def test_dependence_graph_rejects_unestablished_pair() -> None: + transitions, pairs = _graph_rows() + producer, consumer = transitions + invalid = ( + ActionTransition(producer.action_id, frozenset(), frozenset()), + consumer, + ) + + with pytest.raises(InvalidDependenceGraphError): + ActionDependenceGraph(invalid, pairs, frozenset()) + + +def test_dependence_graph_rejects_requirement_without_supply() -> None: + """Every requirement is reached or explicitly declared at graph entry.""" + + action = ContextualActionId("s1", "module", "module") + carrier = Carrier("orphan") + transitions = (ActionTransition(action, frozenset({carrier}), frozenset()),) + + with pytest.raises(InvalidDependenceGraphError, match="requirement"): + ActionDependenceGraph(transitions, (), frozenset()) + + +def test_dependence_graph_rejects_declared_input_without_consumer() -> None: + """Entry requests must belong to an exact Action requirement.""" + + action = ContextualActionId("s1", "module", "module") + unused = Carrier("unused") + transitions = (ActionTransition(action, frozenset(), frozenset()),) + + with pytest.raises(InvalidDependenceGraphError, match="no consumer"): + ActionDependenceGraph(transitions, (), frozenset({unused})) + + +def test_carrier_identity_preserves_python_types() -> None: + assert Carrier(1) != Carrier(True) + assert Carrier(("value", 1)) == Carrier(("value", 1)) + + +def test_carrier_identity_rejects_mutable_values() -> None: + with pytest.raises(TypeError, match="hashable"): + Carrier(["value"]) # type: ignore[arg-type] From 880bfc30243665cb520839888d7fa6f05a3bedcc Mon Sep 17 00:00:00 2001 From: carloea2 Date: Tue, 1 Sep 2026 17:14:32 -0600 Subject: [PATCH 3/3] feat(py2udf): analyze control scopes --- .../mosaic/analysis/__init__.py | 18 + .../mosaic/analysis/controlwalk.py | 2345 +++++++++++++++++ .../mosaic/analysis/scopes.py | 286 ++ .../mosaic/test_controlwalk.py | 106 + 4 files changed, 2755 insertions(+) create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/analysis/__init__.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/analysis/controlwalk.py create mode 100644 py2udf/src/main/python/python_to_workflow/mosaic/analysis/scopes.py create mode 100644 py2udf/src/test/python/python_to_workflow/mosaic/test_controlwalk.py diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/analysis/__init__.py b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/__init__.py new file mode 100644 index 00000000000..92e2015a2e0 --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Orthogonal analysis passes used by compiler compositions.""" diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/analysis/controlwalk.py b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/controlwalk.py new file mode 100644 index 00000000000..3826ecfc7b1 --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/controlwalk.py @@ -0,0 +1,2345 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Identity-agnostic, path-sensitive traversal of an ActionForest.""" + +from __future__ import annotations + +import ast +from collections.abc import Callable, Hashable, Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Protocol + +from python_to_workflow.mosaic.forest import ( + Action, + ActionForest, + Expression, + ForestBuildError, + Parameter, + ParameterRole, + _Builder, + _Component, + _component_position, +) + +SELF = "SELF" +ENTER = "ENTER" +EXIT = "EXIT" + +type Anchor = str | tuple[str, str] +type Identity = Hashable +type Event = tuple[ + Anchor, + tuple[Identity, ...], + tuple[Identity, ...], + tuple[Identity, ...], +] +type Payload = tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] +type ProducerActions = frozenset[str] +type UnresolvedUse = tuple[str, Identity] +type UnresolvedUses = tuple[UnresolvedUse, ...] + + +def _identity_key(value: object) -> tuple[str, str, str]: + """Order arbitrary hashable identities without requiring cross-type order.""" + + kind = type(value) + return kind.__module__, kind.__qualname__, repr(value) + + +def _identity_row_key(row: tuple[object, ...]) -> tuple[tuple[str, str, str], ...]: + """Order payload rows by their structural identity.""" + + return tuple(_identity_key(value) for value in row) + + +@dataclass(frozen=True) +class ControlWalkResult: + """Canonical payload plus exact use occurrences that may be unbound.""" + + payload: Payload + unresolved_uses: UnresolvedUses + + +@dataclass(frozen=True) +class StatementPostState: + """Writes owned by one complete statement and its normal completions.""" + + root: str + may_write: tuple[Identity, ...] + must_write: tuple[Identity, ...] + deletes: tuple[Identity, ...] + + def __post_init__(self) -> None: + rows = (self.may_write, self.must_write, self.deletes) + if any(row != tuple(sorted(set(row), key=_identity_key)) for row in rows): + raise ValueError("statement post-state identities must be canonical") + if not set(self.must_write) <= set(self.may_write): + raise ValueError("must-write identities must also be may-write identities") + + +_WRITE_MARKER = "statement-write" +_DELETE_MARKER = "statement-delete" + + +@dataclass(frozen=True) +class _WriteEventExtractor: + """Turn binding definitions and kills into path-visible write markers.""" + + source: EventExtractor + + def __call__(self, action: Action, /) -> tuple[Event, ...]: + rows = [] + for anchor, _uses, defines, kills in self.source(action): + markers = tuple((_WRITE_MARKER, identity) for identity in defines) + markers += tuple((_WRITE_MARKER, identity) for identity in kills) + markers += tuple((_DELETE_MARKER, identity) for identity in kills) + rows.append((anchor, (), markers, ())) + return tuple(rows) + + +@dataclass(frozen=True) +class _DefinitionState: + """Possible producers plus whether one legal path remains unbound.""" + + producers: ProducerActions + may_be_unbound: bool + + +type State = dict[Identity, _DefinitionState] +type EntryState = Mapping[Identity, frozenset[str]] +type Outcomes = dict[str, State] +type _DefinitionCounts = dict[Identity, int] +type EndpointRelation = tuple[str, str] +type OwnedEndpointRelation = tuple[str, str, str] +type RepeatedDecisionIndex = Mapping[str, frozenset[str]] +type _ActionNodeIndex = Mapping[str, ast.AST] +type _ActionInventory = frozenset[str] +type _ExtractedActionIds = set[str] +type _DormantParameters = frozenset[str] +type _EvaluationIndices = Mapping[str, int] + + +class EventExtractor(Protocol): + """Return canonical anchored uses, definitions, and kills for an Action.""" + + def __call__(self, action: Action, /) -> tuple[Event, ...]: ... + + +@dataclass(frozen=True) +class ControlWalkProgram: + """Source-exact structural control metadata reusable across flow queries.""" + + forest: ActionForest + _nodes: _ActionNodeIndex + action_ids: _ActionInventory + dormant: _DormantParameters + indices: _EvaluationIndices + + +class InvalidAnchorError(ValueError): + """An event anchor does not belong to the Action it labels.""" + + +def controlwalk(forest: ActionForest, events: EventExtractor) -> Payload: + """Walk legal paths and return canonical private flow facts.""" + + return _Walk(compile_controlwalk(forest), events).run() + + +def compile_controlwalk(forest: ActionForest, /) -> ControlWalkProgram: + """Compile immutable control structure once for repeated analysis walks.""" + + forest.validate() + builder = _validated_builder(forest) + _demanded, dormant, indices = _metadata(builder) + nodes = MappingProxyType(dict(builder.action_nodes)) + return ControlWalkProgram( + forest, + nodes, + frozenset(nodes), + dormant, + MappingProxyType(dict(indices)), + ) + + +def controlwalk_region( + forest: ActionForest, + roots: tuple[str, ...], + incoming: EntryState, + events: EventExtractor, + program: ControlWalkProgram | None = None, +) -> Payload: + """Walk one real lexical region through the same path authority. + + ``incoming`` contains raw analysis identities and their alternative source + Actions. It is analysis state only: the walk still emits no Carrier. + """ + + return controlwalk_region_result(forest, roots, incoming, events, program).payload + + +def controlwalk_region_result( + forest: ActionForest, + roots: tuple[str, ...], + incoming: EntryState, + events: EventExtractor, + program: ControlWalkProgram | None = None, +) -> ControlWalkResult: + """Walk one lexical region and retain path evidence per use occurrence.""" + + compiled = _control_program(forest, program) + if roots != tuple(dict.fromkeys(roots)) or not set(roots) <= compiled.action_ids: + raise ValueError("control-walk region roots must be unique forest Actions") + seeded = { + identity: frozenset(producers) for identity, producers in incoming.items() + } + if any(not producers <= compiled.action_ids for producers in seeded.values()): + raise ValueError("control-walk region seed references an unknown Action") + return _Walk(compiled, events).run_region_result(roots, seeded) + + +def statement_post_states( + forest: ActionForest, + roots: tuple[str, ...], + events: EventExtractor, + program: ControlWalkProgram | None = None, +) -> tuple[StatementPostState, ...]: + """Derive may/must writes from the same path authority in linear work.""" + + compiled = _control_program(forest, program) + if roots != tuple(dict.fromkeys(roots)) or not set(roots) <= compiled.action_ids: + raise ValueError("statement post-state roots must be unique forest Actions") + walk = _Walk(compiled, _WriteEventExtractor(events)) + return tuple(_statement_post_state(walk, root) for root in roots) + + +def _statement_post_state(walk: _Walk, root: str) -> StatementPostState: + """Profile one disjoint root without rebuilding its control program.""" + + before = set(walk.definitions) + walk._collect_region_inventory((root,)) + inventory = walk.definitions - before + normal = walk._action(root, {}).get("normal", {}) + may_write = _marked_identities(inventory, _WRITE_MARKER) + must_write = tuple( + sorted( + ( + marker[1] + for marker, state in normal.items() + if _is_marker(marker, _WRITE_MARKER) and not state.may_be_unbound + ), + key=_identity_key, + ) + ) + return StatementPostState( + root, + may_write, + must_write, + _marked_identities(inventory, _DELETE_MARKER), + ) + + +def _marked_identities( + rows: set[tuple[str, Identity]], + kind: str, +) -> tuple[Identity, ...]: + """Decode one canonical marker kind from collected definition rows.""" + + return tuple( + sorted( + {marker[1] for _action, marker in rows if _is_marker(marker, kind)}, + key=_identity_key, + ) + ) + + +def _is_marker(value: object, kind: str) -> bool: + return isinstance(value, tuple) and len(value) == 2 and value[0] == kind + + +def _control_program( + forest: ActionForest, program: ControlWalkProgram | None +) -> ControlWalkProgram: + """Accept only a program compiled for this exact immutable forest.""" + + if program is None: + return compile_controlwalk(forest) + if program.forest is not forest: + raise ValueError("control-walk program belongs to another ActionForest") + return program + + +def region_completion_actions( + forest: ActionForest, + roots: tuple[str, ...], + program: ControlWalkProgram | None = None, +) -> tuple[tuple[str, str], ...]: + """Return exact terminal Action ids by generic completion kind. + + The same protocol walk owns both order and completion propagation. A + private marker records only the last Action reached on each legal path; + it does not become a fact, Carrier, or second control authority. + """ + + marker = ("terminal",) + + def events(action: Action) -> tuple[Event, ...]: + return ((SELF, (), (marker,), ()),) + + compiled = _control_program(forest, program) + walk = _Walk(compiled, events) + outcomes = walk._sequence(roots, {}) + return tuple( + sorted( + (kind, producer) + for kind, state in outcomes.items() + for producer in _state_producers(state, marker) + ) + ) + + +def _state_producers(state: State, identity: Identity) -> ProducerActions: + """Read possible producers without exposing internal boundness state.""" + + row = state.get(identity) + return frozenset() if row is None else row.producers + + +def control_region_owners(forest: ActionForest) -> tuple[str, ...]: + """Derive real Actions that own structured runtime control regions.""" + + nodes = _validated_builder(forest).action_nodes + region_types = ( + ast.BoolOp, + ast.For, + ast.If, + ast.IfExp, + ast.Match, + ast.Try, + ast.While, + ast.With, + ) + return tuple( + sorted( + action_id + for action_id, node in nodes.items() + if isinstance(node, region_types) + ) + ) + + +def completion_region_owners(forest: ActionForest) -> tuple[str, ...]: + """Return source owners whose runtime semantics thread completions.""" + + nodes = _validated_builder(forest).action_nodes + return tuple( + sorted( + action_id + for action_id, node in nodes.items() + if isinstance(node, (ast.Try, ast.With)) + ) + ) + + +def locally_handled_name_error_uses( + forest: ActionForest, + uses: UnresolvedUses, + reaching: tuple[tuple[object, ...], ...], + /, +) -> UnresolvedUses: + """Return exact unbound-name occurrences handled by an enclosing try. + + This is control evidence, not a binding heuristic: only occurrences in a + real try body qualify, and finally/else/handler regions remain outside the + handler they follow. + """ + + nodes = _validated_builder(forest).action_nodes + unresolved = frozenset(uses) + reached = frozenset( + (consumer, identity) for _producer, consumer, identity in reaching + ) + evidence = _HandlerEvidence( + forest, + nodes, + {id(node): action_id for action_id, node in nodes.items()}, + unresolved, + reached, + ) + handled: dict[str, bool] = {} + + def qualifies(action_id: str) -> bool: + if action_id not in handled: + handled[action_id] = _has_enclosing_name_error_handler(evidence, action_id) + return handled[action_id] + + return tuple(row for row in uses if qualifies(row[0])) + + +@dataclass(frozen=True) +class _HandlerEvidence: + """Immutable indices for exact exception-selector certification.""" + + forest: ActionForest + nodes: Mapping[str, ast.AST] + action_by_node: Mapping[int, str] + unresolved: frozenset[tuple[str, Hashable]] + reached: frozenset[tuple[str, Hashable]] + + +def _has_enclosing_name_error_handler( + evidence: _HandlerEvidence, + action_id: str, +) -> bool: + current = action_id + while (parameter_id := evidence.forest.parent_parameter(current)) is not None: + parameter = evidence.forest.parameter(parameter_id) + owner = parameter.owner + node = evidence.nodes[owner] + if ( + isinstance(node, (ast.Try, ast.TryStar)) + and parameter.name == "body" + and any( + _catches_name_error( + handler.type, + evidence.action_by_node, + evidence.unresolved, + evidence.reached, + ) + for handler in node.handlers + ) + ): + return True + current = owner + return False + + +def _catches_name_error( + node: ast.expr | None, + action_by_node: Mapping[int, str], + unresolved: frozenset[tuple[str, Hashable]], + reached: frozenset[tuple[str, Hashable]], +) -> bool: + """Accept a named selector only when binding flow proves builtin fallback.""" + + if node is None: + return True + if isinstance(node, ast.Name): + if node.id not in {"NameError", "Exception", "BaseException"}: + return False + action_id = action_by_node.get(id(node)) + occurrence = (action_id, ("module", node.id)) + return ( + action_id is not None + and occurrence in unresolved + and occurrence not in reached + ) + if isinstance(node, ast.Tuple): + return any( + _catches_name_error(item, action_by_node, unresolved, reached) + for item in node.elts + ) + return False + + +@dataclass(frozen=True) +class CompletionStage: + """Source Actions plus exact entry and exit frontiers of one SUITE stage.""" + + parameter_id: str + actions: frozenset[str] + entries: tuple[str, ...] + exits: tuple[str, ...] + + +@dataclass(frozen=True) +class ChoiceRegion: + """One source choice, its complete inventory, and terminal frontier.""" + + owner: str + members: frozenset[str] + exits: tuple[str, ...] + + +@dataclass(frozen=True) +class ExceptionHandler: + """One source-ordered exception selector and its owned handler body.""" + + selector: CompletionStage | None + body: CompletionStage + + +@dataclass(frozen=True) +class CompletionRegion: + """The source-exact stages governed by one Python try Action.""" + + owner: str + body: CompletionStage + handlers: tuple[ExceptionHandler, ...] + orelse: CompletionStage | None + finalbody: CompletionStage | None + + +@dataclass(frozen=True) +class WithRegion: + """One source-exact context-manager owner and its protected body stage.""" + + owner: str + body: CompletionStage + + +@dataclass(frozen=True) +class IterationRegion: + """One loop owner with its repeated body and exhaustion-only else stage.""" + + owner: str + body: CompletionStage + orelse: CompletionStage | None + + +@dataclass(frozen=True) +class IterationProtocolIndex: + """Immutable loop containment shared by every structural projection.""" + + regions: tuple[IterationRegion, ...] + members: Mapping[str, frozenset[str]] + body_owners: Mapping[str, tuple[str, ...]] + admission_owner: Mapping[str, str] + + +@dataclass(frozen=True) +class ControlRelationProjection: + """Precise control rows and exact baseline world relations they replace.""" + + owned_relations: tuple[OwnedEndpointRelation, ...] + retired_relations: tuple[EndpointRelation, ...] + + +def control_relations( + forest: ActionForest, relations: tuple[tuple[str, str], ...] +) -> ControlRelationProjection: + """Return owner, producer, consumer rows for every control Carrier. + + Ordinary regions mirror exact dependency endpoints. ``try`` regions also + emit standalone owner-entry and clause-handoff bases so every ordinary + continuation is solver-visible before coloring. Their source-ordered + spine reaches the first selector, bypasses ``else`` after a handler, and + reaches ``finally`` without a raising-site × handler expansion. + """ + + if relations != tuple(sorted(set(relations))): + raise ValueError("control endpoint relations must be canonical") + builder = _validated_builder(forest) + entries = _EntryActions(forest, builder) + tries = completion_regions(forest, builder, entries) + contexts = with_regions(forest, builder, entries) + try_owners = frozenset(region.owner for region in tries) + ordinary = frozenset(control_region_owners(forest)) - try_owners + rows = _ordinary_control_relations(forest, relations, ordinary) + rows = {row for row in rows if not _crosses_enclosing_try(row, tries)} + replaced: set[EndpointRelation] = set() + for region in tries: + rows.update(_try_control_relations(forest, relations, region)) + rows.update( + _try_physical_route_bases(forest, relations, region, tries, entries) + ) + replaced.update(_try_replaced_relations(relations, region)) + for region in contexts: + rows.update(_with_control_relations(region)) + return ControlRelationProjection(tuple(sorted(rows)), tuple(sorted(replaced))) + + +def _crosses_enclosing_try( + row: OwnedEndpointRelation, regions: tuple[CompletionRegion, ...] +) -> bool: + """Stop a nested region's control identity at its enclosing try boundary.""" + + owner, producer, consumer = row + for region in regions: + stages = _try_stages(region) + actions = _try_region_actions(region, stages) + if owner in actions and {producer, consumer} <= actions: + return not _same_stage(producer, consumer, stages) + return False + + +def _ordinary_control_relations( + forest: ActionForest, + relations: tuple[tuple[str, str], ...], + owners: frozenset[str], +) -> set[tuple[str, str, str]]: + """Mirror continuity and admit every independently scheduled SUITE root.""" + + result: set[tuple[str, str, str]] = set() + for producer, consumer in relations: + ancestors = _endpoint_ancestors(forest, producer, consumer) + result.update((owner, producer, consumer) for owner in owners & ancestors) + loop_owners = frozenset(iteration_region_owners(forest)) + for owner in owners - loop_owners: + result.update( + (owner, owner, target) + for target in _suite_admission_roots(forest, owner, relations) + ) + return result + + +def _suite_admission_roots( + forest: ActionForest, + owner: str, + relations: tuple[EndpointRelation, ...], + stage: frozenset[str] | None = None, +) -> tuple[str, ...]: + """Find physical SUITE roots after analysis has discharged safe ordering. + + Source order alone is insufficient here: eager-safe siblings may no longer + have an ordinary edge between them, but each still needs the dynamic + region's admission token. Nested SUITEs belong to their own nearest owner; + non-SUITE parameters of a nested owner remain part of the enclosing stage. + """ + + candidates = frozenset( + action.id + for action in forest.actions + if ( + _nearest_suite_owner(forest, action.id) == owner + or stage is not None + and forest.parent_action(action.id) == owner + ) + and (stage is None or action.id in stage) + ) + inventory = candidates if stage is None else stage + internal_consumers = frozenset( + consumer + for producer, consumer in relations + if producer in inventory and consumer in inventory + ) + return tuple(sorted(candidates - internal_consumers)) + + +def _nearest_suite_owner(forest: ActionForest, action_id: str) -> str | None: + """Return the owner of the nearest source SUITE containing one Action.""" + + current = action_id + while (parameter_id := forest.parent_parameter(current)) is not None: + parameter = forest.parameter(parameter_id) + if parameter.role is ParameterRole.SUITE: + return parameter.owner + current = parameter.owner + return None + + +def _endpoint_ancestors( + forest: ActionForest, producer: str, consumer: str +) -> frozenset[str]: + """Return both endpoint ownership spines, including the endpoints.""" + + result: set[str] = set() + for endpoint in (producer, consumer): + current: str | None = endpoint + while current is not None: + result.add(current) + current = forest.parent_action(current) + return frozenset(result) + + +def completion_regions( + forest: ActionForest, + builder: _Builder | None = None, + entries: _EntryActions | None = None, +) -> tuple[CompletionRegion, ...]: + """Build every try protocol once from source-exact forest Parameters.""" + + builder = _validated_builder(forest) if builder is None else builder + entries = _EntryActions(forest, builder) if entries is None else entries + owners = tuple( + sorted( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, ast.Try) + ) + ) + return tuple(_try_region(forest, owner, entries) for owner in owners) + + +def completion_resumption_exits( + forest: ActionForest, + region: CompletionRegion, + stage: CompletionStage, + regions: tuple[CompletionRegion, ...], +) -> tuple[str, ...]: + """Derive the post-passthrough physical frontier of one try stage.""" + + nested = tuple( + (candidate, _try_region_actions(candidate, _try_stages(candidate))) + for candidate in regions + if candidate.owner != region.owner and candidate.owner in stage.actions + ) + parameter = forest.parameter(stage.parameter_id) + physical_roots = parameter.actions[-1:] + semantic_exits = tuple( + action_id + for _completion, action_id in region_completion_actions(forest, physical_roots) + ) + exits = { + replacement + for source in semantic_exits + for replacement in _resumption_exit(source, nested) + } + return tuple(sorted(exits)) + + +def _resumption_exit( + source: str, + nested: tuple[tuple[CompletionRegion, frozenset[str]], ...], +) -> tuple[str, ...]: + """Select the immediate nested owner containing one semantic exit.""" + + owners = tuple(row for row in nested if source in row[1]) + if not owners: + return (source,) + region, _members = max(owners, key=lambda row: len(row[1])) + return _try_stages(region)[-1].exits + + +def with_regions( + forest: ActionForest, + builder: _Builder | None = None, + entries: _EntryActions | None = None, +) -> tuple[WithRegion, ...]: + """Build every context-manager suspension from source-exact Parameters.""" + + builder = _validated_builder(forest) if builder is None else builder + entries = _EntryActions(forest, builder) if entries is None else entries + owners = tuple( + sorted( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, ast.With) + ) + ) + return tuple(_with_region(forest, owner, entries) for owner in owners) + + +def _with_region( + forest: ActionForest, owner: str, entries: _EntryActions +) -> WithRegion: + """Name one protected body without duplicating context-item structure.""" + + body = next( + forest.parameter(parameter_id) + for parameter_id in forest.action(owner).parameters + if forest.parameter(parameter_id).name == "body" + ) + return WithRegion(owner, _completion_stage(forest, body, entries)) + + +def _with_control_relations(region: WithRegion) -> set[OwnedEndpointRelation]: + """Thread entry forward and body completion back to the suspended owner.""" + + entries = {(region.owner, region.owner, target) for target in region.body.entries} + returns = {(region.owner, source, region.owner) for source in region.body.exits} + return entries | returns + + +def _try_region( + forest: ActionForest, owner: str, entries: _EntryActions +) -> CompletionRegion: + """Name the ordered clauses owned by one try Action.""" + + parameters = tuple( + forest.parameter(parameter_id) + for parameter_id in forest.action(owner).parameters + ) + by_name = {parameter.name: parameter for parameter in parameters} + handler_prefixes = tuple( + parameter.name.removesuffix("_body") + for parameter in parameters + if parameter.name.startswith("handler") and parameter.name.endswith("_body") + ) + handlers = tuple( + ExceptionHandler( + _optional_stage(forest, by_name.get(f"{prefix}_type"), entries), + _completion_stage(forest, by_name[f"{prefix}_body"], entries), + ) + for prefix in handler_prefixes + ) + return CompletionRegion( + owner, + _completion_stage(forest, by_name["body"], entries), + handlers, + _optional_stage(forest, by_name.get("orelse"), entries), + _optional_stage(forest, by_name.get("finalbody"), entries), + ) + + +def _optional_stage( + forest: ActionForest, + parameter: Parameter | None, + entries: _EntryActions, +) -> CompletionStage | None: + """Build an optional clause without an empty sentinel record.""" + + return None if parameter is None else _completion_stage(forest, parameter, entries) + + +def _completion_stage( + forest: ActionForest, parameter: Parameter, entries: _EntryActions +) -> CompletionStage: + """Derive exact entry and terminal Actions for one real Parameter region.""" + + actions = frozenset(_descendants(forest, parameter.actions)) + starts = entries.of(parameter.actions[0]) if parameter.actions else () + exits = tuple( + sorted( + { + action_id + for _completion, action_id in region_completion_actions( + forest, parameter.actions + ) + } + ) + ) + return CompletionStage(parameter.id, actions, starts, exits) + + +def _try_control_relations( + forest: ActionForest, + relations: tuple[tuple[str, str], ...], + region: CompletionRegion, +) -> set[tuple[str, str, str]]: + """Project one try into companions plus an O(clauses) control spine.""" + + stages = _try_stages(region) + region_actions = _try_region_actions(region, stages) + result = { + (region.owner, producer, consumer) + for producer, consumer in relations + if _try_companion_relation( + producer, consumer, region.owner, region_actions, stages + ) + } + result.update(_try_entry_handoffs(forest, relations, region)) + result.update(_try_body_handoffs(forest, relations, region)) + result.update(_try_handler_handoffs(forest, relations, region)) + return result + + +def _try_companion_relation( + producer: str, + consumer: str, + owner: str, + actions: frozenset[str], + stages: tuple[CompletionStage, ...], +) -> bool: + """Keep real local continuity without turning data ingress into control.""" + + return ( + _same_stage(producer, consumer, stages) + or consumer == owner + or producer in actions + and consumer not in actions + ) + + +def _try_entry_handoffs( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, +) -> set[OwnedEndpointRelation]: + """Enter the protected body from its owner, never from a data producer.""" + + return { + (region.owner, region.owner, target) + for target in completion_stage_admission_roots( + forest, relations, region, region.body + ) + } + + +def completion_stage_admission_roots( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + stage: CompletionStage, +) -> tuple[str, ...]: + """Return every post-discharge physical root in one try clause.""" + + return _suite_admission_roots(forest, region.owner, relations, stage.actions) + + +def _try_physical_route_bases( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + regions: tuple[CompletionRegion, ...], + entries: _EntryActions, +) -> set[OwnedEndpointRelation]: + """Expose every ordinary completion hop before physical placement.""" + + stages = _try_stages(region) + exits = tuple( + completion_resumption_exits(forest, region, stage, regions) for stage in stages + ) + result = { + (region.owner, source, target) + for index, stage_exits in enumerate(exits[:-1]) + for source in stage_exits + for target in completion_stage_admission_roots( + forest, relations, region, stages[index + 1] + ) + } + continuations = _try_continuation_targets(forest, relations, region, entries) + result.update( + (region.owner, source, target) + for source in exits[-1] + for target in continuations + ) + result.update(_try_nested_route_bases(forest, relations, region, regions)) + return result + + +def _try_continuation_targets( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + entries: _EntryActions, +) -> tuple[str, ...]: + """Project every external continuation through the try's final frontier.""" + + actions = _try_region_actions(region, _try_stages(region)) + external = { + consumer + for producer, consumer in relations + if producer in actions and consumer not in actions + } + external.update(_following_parameter_entries(forest, region.owner, entries)) + return tuple(sorted(external)) + + +def _following_parameter_entries( + forest: ActionForest, action_id: str, entries: _EntryActions +) -> tuple[str, ...]: + """Return the first lexical sibling after one completed structured Action.""" + + parameter_id = forest.parent_parameter(action_id) + siblings = ( + forest.roots if parameter_id is None else forest.parameter(parameter_id).actions + ) + following = siblings[siblings.index(action_id) + 1 :] + return () if not following else entries.of(following[0]) + + +def _try_nested_route_bases( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + regions: tuple[CompletionRegion, ...], +) -> set[OwnedEndpointRelation]: + """Expose exact final-frontier hops out of nested completions in one stage.""" + + result: set[OwnedEndpointRelation] = set() + for stage in _try_stages(region): + nested = tuple( + sorted( + ( + (candidate, _try_region_actions(candidate, _try_stages(candidate))) + for candidate in regions + if candidate != region and candidate.owner in stage.actions + ), + key=lambda row: (len(row[1]), row[0].owner), + ) + ) + for producer, consumer in relations: + candidates = tuple( + candidate + for candidate, members in nested + if producer in members and consumer not in members + ) + if not candidates or consumer not in stage.actions: + continue + candidate = candidates[0] + final = _try_stages(candidate)[-1] + result.update( + (region.owner, source, consumer) + for source in completion_resumption_exits( + forest, candidate, final, regions + ) + ) + return result + + +def _try_replaced_relations( + relations: tuple[EndpointRelation, ...], region: CompletionRegion +) -> set[EndpointRelation]: + """Identify redundant routes inside one try region.""" + + stages = _try_stages(region) + actions = _try_region_actions(region, stages) + return { + relation + for relation in relations + if set(relation) <= actions and not _same_stage(*relation, stages) + } + + +def _try_region_actions( + region: CompletionRegion, stages: tuple[CompletionStage, ...] +) -> frozenset[str]: + """Return one owner's complete source Action inventory.""" + + return frozenset( + {region.owner, *(action for stage in stages for action in stage.actions)} + ) + + +def _try_stages(region: CompletionRegion) -> tuple[CompletionStage, ...]: + """Flatten one named try region without losing handler order.""" + + handlers = tuple( + stage + for handler in region.handlers + for stage in (handler.selector, handler.body) + if stage is not None + ) + suffix = tuple( + stage for stage in (region.orelse, region.finalbody) if stage is not None + ) + return region.body, *handlers, *suffix + + +def _same_stage( + producer: str, consumer: str, stages: tuple[CompletionStage, ...] +) -> bool: + """Recognize one dependency whose endpoints share a real clause.""" + + return any( + producer in stage.actions and consumer in stage.actions for stage in stages + ) + + +def _try_body_handoffs( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, +) -> set[tuple[str, str, str]]: + """Connect body terminals to handler search, else, and finalization.""" + + first_handler = _handler_admission_roots( + forest, relations, region, region.handlers[:1] + ) + orelse = _optional_stage_roots(forest, relations, region, region.orelse) + finalbody = _optional_stage_roots(forest, relations, region, region.finalbody) + targets = frozenset((*first_handler, *orelse, *finalbody)) + return { + (region.owner, source, target) + for source in region.body.exits + for target in targets + } + + +def _try_handler_handoffs( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, +) -> set[tuple[str, str, str]]: + """Chain nonmatches and send every matched outcome around else.""" + + finalbody = _optional_stage_roots(forest, relations, region, region.finalbody) + result: set[tuple[str, str, str]] = set() + for index, handler in enumerate(region.handlers): + following = _handler_admission_roots( + forest, relations, region, region.handlers[index + 1 : index + 2] + ) + selector_targets = frozenset( + ( + *completion_stage_admission_roots( + forest, relations, region, handler.body + ), + *following, + *finalbody, + ) + ) + selector_exits = () if handler.selector is None else handler.selector.exits + result.update( + (region.owner, source, target) + for source in selector_exits + for target in selector_targets + ) + result.update( + (region.owner, source, target) + for source in handler.body.exits + for target in finalbody + ) + if region.orelse is not None: + result.update( + (region.owner, source, target) + for source in region.orelse.exits + for target in finalbody + ) + return result + + +def _optional_stage_roots( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + stage: CompletionStage | None, +) -> tuple[str, ...]: + """Return physical roots for one optional real clause.""" + + return ( + () + if stage is None + else completion_stage_admission_roots(forest, relations, region, stage) + ) + + +def _handler_admission_roots( + forest: ActionForest, + relations: tuple[EndpointRelation, ...], + region: CompletionRegion, + handlers: tuple[ExceptionHandler, ...], +) -> tuple[str, ...]: + """Return selector roots, or body roots for one bare handler.""" + + return tuple( + root + for handler in handlers + for root in completion_stage_admission_roots( + forest, + relations, + region, + handler.body if handler.selector is None else handler.selector, + ) + ) + + +def _handler_entries(handlers: tuple[ExceptionHandler, ...]) -> tuple[str, ...]: + """Return the selector entry, or the body entry for one bare handler.""" + + return tuple( + entry + for handler in handlers + for entry in ( + handler.body.entries + if handler.selector is None + else handler.selector.entries + ) + ) + + +def choice_passthrough_entries( + forest: ActionForest, + builder: _Builder | None = None, +) -> tuple[tuple[str, str, str], ...]: + """Return completion-to-continuation entries for split choices. + + These rows are physical source-suppression routes, not dependence facts. + The control protocol owns the target: it is the first Action evaluated by + the source statement following the nearest enclosing ``if`` or ``match``. + """ + + builder = _validated_builder(forest) if builder is None else builder + entries = _EntryActions(forest, builder) + choices = frozenset( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, (ast.If, ast.Match)) + ) + completions = ( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, (ast.Break, ast.Continue, ast.Raise, ast.Return)) + ) + rows = ( + (owner, completion, target) + for completion in completions + if (owner := _nearest_owner(forest, completion, choices)) is not None + for successor in _following_siblings(forest, owner, choices)[:1] + for target in entries.of(successor) + ) + return tuple(sorted(set(rows))) + + +def choice_regions( + forest: ActionForest, builder: _Builder | None = None +) -> tuple[ChoiceRegion, ...]: + """Derive exact `if`/`match` inventories through the path authority.""" + + builder = _validated_builder(forest) if builder is None else builder + owners = tuple( + sorted( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, (ast.If, ast.Match)) + ) + ) + return tuple( + ChoiceRegion( + owner, + frozenset(_descendants(forest, (owner,))), + tuple( + sorted( + { + action_id + for _kind, action_id in region_completion_actions( + forest, (owner,) + ) + } + ) + ), + ) + for owner in owners + ) + + +def _nearest_owner( + forest: ActionForest, action_id: str, owners: frozenset[str] +) -> str | None: + """Find the nearest structurally enclosing owner from one fixed family.""" + + current = forest.parent_action(action_id) + while current is not None and current not in owners: + current = forest.parent_action(current) + return current + + +def _following_siblings( + forest: ActionForest, action_id: str, choices: frozenset[str] +) -> tuple[str, ...]: + """Find the next source Action through only enclosing choice continuations.""" + + while True: + parameter_id = forest.parent_parameter(action_id) + siblings = ( + forest.roots + if parameter_id is None + else forest.parameter(parameter_id).actions + ) + following = siblings[siblings.index(action_id) + 1 :] + if following: + return following + action_id = forest.parent_action(action_id) + if action_id not in choices: + return () + + +class _EntryActions: + """Memoized first-evaluation query derived from the control protocol.""" + + def __init__(self, forest: ActionForest, builder: _Builder) -> None: + self.forest = forest + self.nodes = builder.action_nodes + _demanded, self.dormant, self.indices = _metadata(builder) + self.memo: dict[str, tuple[str, ...]] = {} + + def of(self, action_id: str) -> tuple[str, ...]: + """Return the first executable Actions for one source Action.""" + + if action_id not in self.memo: + self.memo[action_id] = self._derive(action_id) + return self.memo[action_id] + + def _derive(self, action_id: str) -> tuple[str, ...]: + node = self.nodes[action_id] + if isinstance(node, ast.Try): + return (action_id,) + parameters = tuple( + sorted( + ( + self.forest.parameter(parameter_id) + for parameter_id in self.forest.action(action_id).parameters + if parameter_id not in self.dormant + and self.forest.parameter(parameter_id).role + is not ParameterRole.SUITE + ), + key=lambda parameter: self.indices[parameter.id], + ) + ) + first = next((parameter for parameter in parameters if parameter.actions), None) + if first is None: + return (action_id,) + return self.of(first.actions[0]) + + +def demanded_parameter_entries( + forest: ActionForest, +) -> tuple[tuple[str, str, str], ...]: + """Return canonical owner, entry Action, and demanded Parameter ids.""" + + builder = _validated_builder(forest) + demanded, _dormant, _indices = _metadata(builder) + entries = _EntryActions(forest, builder) + parameters = (forest.parameter(parameter_id) for parameter_id in demanded) + return tuple( + sorted( + (parameter.owner, entry, parameter.id) + for parameter in parameters + if parameter.role is not ParameterRole.SUITE + for action_id in parameter.actions + for entry in entries.of(action_id) + ) + ) + + +def iteration_region_owners(forest: ActionForest) -> tuple[str, ...]: + """Return canonical loop owners from the path authority.""" + + return tuple(region.owner for region in iteration_protocol_index(forest).regions) + + +def iteration_protocol_index(forest: ActionForest) -> IterationProtocolIndex: + """Build loop regions and nested membership once per immutable forest.""" + + builder = _validated_builder(forest) + entries = _EntryActions(forest, builder) + regions = tuple( + _iteration_region(forest, owner, entries) + for owner in sorted( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, (ast.For, ast.While)) + ) + ) + members = { + region.owner: frozenset(_descendants(forest, (region.owner,))) + for region in regions + } + order = {owner: (-len(actions), owner) for owner, actions in members.items()} + owners = frozenset(members) + return IterationProtocolIndex( + regions, + MappingProxyType(members), + _iteration_owner_index( + tuple((region.owner, region.body.actions) for region in regions), order + ), + MappingProxyType( + { + action.id: owner + for action in forest.actions + for owner in (_nearest_suite_owner(forest, action.id),) + if owner in owners + } + ), + ) + + +def _iteration_owner_index( + rows: tuple[tuple[str, frozenset[str]], ...], + order: Mapping[str, tuple[int, str]], +) -> Mapping[str, tuple[str, ...]]: + """Invert owner membership in deterministic outer-to-inner order.""" + + mutable: dict[str, list[str]] = {} + for owner, actions in rows: + for action_id in actions: + mutable.setdefault(action_id, []).append(owner) + return MappingProxyType( + { + action_id: tuple(sorted(owners, key=order.__getitem__)) + for action_id, owners in mutable.items() + } + ) + + +def iteration_regions( + forest: ActionForest, + builder: _Builder | None = None, + entries: _EntryActions | None = None, +) -> tuple[IterationRegion, ...]: + """Build every loop SUITE protocol once from source-exact Parameters.""" + + if builder is None and entries is None: + return iteration_protocol_index(forest).regions + builder = _validated_builder(forest) if builder is None else builder + entries = _EntryActions(forest, builder) if entries is None else entries + owners = tuple( + sorted( + action_id + for action_id, node in builder.action_nodes.items() + if isinstance(node, (ast.For, ast.While)) + ) + ) + return tuple(_iteration_region(forest, owner, entries) for owner in owners) + + +def _iteration_region( + forest: ActionForest, owner: str, entries: _EntryActions +) -> IterationRegion: + """Name repeated and exhaustion-only stages without encoding an order.""" + + parameters = tuple( + forest.parameter(parameter_id) + for parameter_id in forest.action(owner).parameters + ) + by_name = {parameter.name: parameter for parameter in parameters} + return IterationRegion( + owner, + _completion_stage(forest, by_name["body"], entries), + _optional_stage(forest, by_name.get("orelse"), entries), + ) + + +def iteration_relations( + forest: ActionForest, + relations: tuple[tuple[str, str], ...], +) -> tuple[tuple[tuple[str, str, str], ...], tuple[tuple[str, str, str], ...]]: + """Partition exact endpoint relations into loop-body entry and return.""" + + canonical = tuple(sorted(set(relations))) + if relations != canonical: + raise ValueError("iteration endpoint relations must be canonical") + index = iteration_protocol_index(forest) + by_owner = {region.owner: region for region in index.regions} + entries: set[OwnedEndpointRelation] = set() + returns: set[OwnedEndpointRelation] = set() + for producer, consumer in relations: + entries.update( + (owner, producer, consumer) + for owner in index.body_owners.get(consumer, ()) + for region in (by_owner[owner],) + if producer in index.members[owner] and producer not in region.body.actions + ) + returns.update( + (owner, producer, consumer) + for owner in index.body_owners.get(producer, ()) + for region in (by_owner[owner],) + if consumer in index.members[owner] and consumer not in region.body.actions + ) + return tuple(sorted(entries)), tuple(sorted(returns)) + + +def iteration_admission_relations( + forest: ActionForest, + physical_relations: tuple[EndpointRelation, ...], + structural_entries: tuple[OwnedEndpointRelation, ...], +) -> tuple[OwnedEndpointRelation, ...]: + """Admit physical SUITE roots not already entered structurally.""" + + if physical_relations != tuple(sorted(set(physical_relations))): + raise ValueError("physical iteration relations must be canonical") + index = iteration_protocol_index(forest) + entered = frozenset( + (owner, consumer) for owner, _producer, consumer in structural_entries + ) + internal = frozenset( + (owner, consumer) + for producer, consumer in physical_relations + for owner in (index.admission_owner.get(producer),) + if owner is not None and index.admission_owner.get(consumer) == owner + ) + return tuple( + sorted( + (owner, owner, target) + for target, owner in index.admission_owner.items() + if (owner, target) not in internal and (owner, target) not in entered + ) + ) + + +def iteration_feedback_relations( + forest: ActionForest, relations: tuple[tuple[str, str], ...] +) -> tuple[tuple[str, str, str], ...]: + """Return exact endpoint relations satisfiable only after loop re-entry.""" + + if relations != tuple(sorted(set(relations))): + raise ValueError("iteration endpoint relations must be canonical") + by_consumer: dict[str, list[tuple[str, str]]] = {} + by_producer: dict[str, list[tuple[str, str]]] = {} + for relation in relations: + by_producer.setdefault(relation[0], []).append(relation) + by_consumer.setdefault(relation[1], []).append(relation) + + def events(action: Action) -> tuple[Event, ...]: + uses = tuple(("relation", *row) for row in by_consumer.get(action.id, ())) + defines = tuple(("relation", *row) for row in by_producer.get(action.id, ())) + return ((SELF, uses, defines, ()),) + + walk = _Walk(compile_controlwalk(forest), events) + walk.run() + repeated = _repeated_decisions(forest) + exhaustion_only = frozenset( + action_id + for region in iteration_regions(forest) + if region.orelse is not None + for action_id in region.orelse.actions + ) + return tuple( + sorted( + (owner, producer, consumer) + for owner, producer, consumer, identity in walk.iteration_feedback + if identity == ("relation", producer, consumer) + and consumer not in exhaustion_only + and not _is_repeated_decision_spine((producer, consumer), repeated) + ) + ) + + +def _repeated_decisions(forest: ActionForest) -> RepeatedDecisionIndex: + """Index loop Parameters reevaluated by their persistent owner.""" + + return { + region.owner: frozenset(_descendants(forest, parameter.actions)) + for region in iteration_regions(forest) + for parameter in ( + next( + ( + forest.parameter(parameter_id) + for parameter_id in forest.action(region.owner).parameters + if forest.parameter(parameter_id).name == "condition" + ), + None, + ), + ) + if parameter is not None + } + + +def _is_repeated_decision_spine( + relation: EndpointRelation, + repeated: RepeatedDecisionIndex, +) -> bool: + """Keep every nested owner/decision spine in its ordinary lazy SCC.""" + + producer, consumer = relation + return any( + (producer == owner and consumer in actions) + or (consumer == owner and producer in actions) + for owner, actions in repeated.items() + ) + + +def _descendants(forest: ActionForest, roots: tuple[str, ...]) -> set[str]: + result: set[str] = set() + pending = list(roots) + while pending: + action_id = pending.pop() + if action_id in result: + continue + result.add(action_id) + pending.extend( + child + for parameter_id in forest.action(action_id).parameters + for child in forest.parameter(parameter_id).actions + ) + return result + + +def _iteration_body(forest: ActionForest, owner: str) -> frozenset[str]: + suites = tuple( + forest.parameter(parameter_id) + for parameter_id in forest.action(owner).parameters + if forest.parameter(parameter_id).role is ParameterRole.SUITE + and forest.parameter(parameter_id).name == "body" + ) + if len(suites) != 1: + raise ForestBuildError(f"loop Action {owner!r} lacks one body SUITE") + return frozenset(_descendants(forest, suites[0].actions)) + + +class _Walk: + def __init__( + self, + program: ControlWalkProgram, + events: EventExtractor, + ) -> None: + self.forest = program.forest + self.dormant = program.dormant + self.indices = program.indices + self.nodes = program._nodes + self.events: dict[tuple[str, Anchor], tuple[tuple[Identity, ...], ...]] = {} + self.definitions: set[tuple[str, Identity]] = set() + self.uses: set[tuple[str, Identity]] = set() + self.reaching: set[tuple[str, str, Identity]] = set() + self.unbound_uses: set[Identity] = set() + self.unbound_occurrences: set[UnresolvedUse] = set() + self.iteration_feedback: set[tuple[str, str, str, Identity]] = set() + self._feedback_context: ( + tuple[str, frozenset[tuple[str, str, Identity]]] | None + ) = None + self._extractor = events + self._extracted: _ExtractedActionIds = set() + + def run(self) -> Payload: + """Walk the complete module region from an empty entry state.""" + + return self.run_region(self.forest.roots, {}) + + def run_region(self, roots: tuple[str, ...], incoming: EntryState) -> Payload: + """Run the collected protocol from an explicit lexical entry state.""" + + return self.run_region_result(roots, incoming).payload + + def run_region_result( + self, roots: tuple[str, ...], incoming: EntryState + ) -> ControlWalkResult: + """Run once and preserve private path evidence beside the codec.""" + + self._collect_region_inventory(roots) + seeded = { + identity: _DefinitionState(producers, False) + for identity, producers in incoming.items() + } + self._sequence(roots, seeded) + return ControlWalkResult( + ( + ( + "definitions", + tuple(sorted(self.definitions, key=_identity_row_key)), + ), + ("uses", tuple(sorted(self.uses, key=_identity_row_key))), + ("reaching", tuple(sorted(self.reaching, key=_identity_row_key))), + ( + "declared_inputs", + tuple(sorted(self.unbound_uses, key=_identity_key)), + ), + ), + tuple(sorted(self.unbound_occurrences, key=_identity_row_key)), + ) + + def _collect_region_inventory(self, roots: tuple[str, ...]) -> None: + """Index lexical events once without pretending every row executes. + + Definitions and uses describe the source inventory of the queried + region. Reaching and unbound evidence are populated separately by + ``_sequence`` over legal control paths. The structural traversal + therefore visits every nested Parameter except a deferred code body; + it neither orders Actions nor changes flow state. + """ + + pending = list(reversed(roots)) + while pending: + action_id = pending.pop() + action = self.forest.action(action_id) + self._collect_action(action) + children = tuple( + child + for parameter_id in action.parameters + if parameter_id not in self.dormant + for child in self.forest.parameter(parameter_id).actions + ) + pending.extend(reversed(children)) + + def _collect_action(self, action: Action) -> None: + """Extract one Action exactly when the control protocol reaches it.""" + + if action.id in self._extracted: + return + self._extracted.add(action.id) + rows = self._extractor(action) + if not isinstance(rows, tuple): + raise TypeError("event extractor must return a tuple") + for row in rows: + self._collect_row(action, row) + + def _collect_row(self, action: Action, row: object) -> None: + if not isinstance(row, tuple) or len(row) != 4: + raise TypeError("each event must be an anchored four-tuple") + anchor, uses, defines, kills = row + self._validate_anchor(action, anchor) + if not all(isinstance(items, tuple) for items in (uses, defines, kills)): + raise TypeError("event uses, defines, and kills must be tuples") + key = (action.id, anchor) + current = self.events.get(key) + # An anchor usually receives one row, so the merge path exists for the + # rare repeat rather than being paid on every event. Set construction + # below is also what validates that every identity is hashable. + if current is None: + self.events[key] = ( + tuple(sorted(set(uses), key=_identity_key)), + tuple(sorted(set(defines), key=_identity_key)), + tuple(sorted(set(kills), key=_identity_key)), + ) + else: + self.events[key] = tuple( + tuple(sorted(set(before) | set(after), key=_identity_key)) + for before, after in zip(current, (uses, defines, kills), strict=True) + ) + self.uses.update((action.id, identity) for identity in uses) + self.definitions.update((action.id, identity) for identity in defines) + + def _validate_anchor(self, action: Action, anchor: Anchor) -> None: + if anchor == SELF: + return + valid = ( + isinstance(anchor, tuple) + and len(anchor) == 2 + and anchor[0] in action.parameters + and anchor[1] in {ENTER, EXIT} + ) + if not valid: + raise InvalidAnchorError( + f"invalid anchor {anchor!r} for Action {action.id}" + ) + + def _sequence(self, action_ids: tuple[str, ...], incoming: State) -> Outcomes: + outcomes: Outcomes = {"normal": dict(incoming)} + for action_id in action_ids: + normal = outcomes.pop("normal", None) + if normal is None: + break + outcomes = _union_outcomes((outcomes, self._action(action_id, normal))) + return outcomes + + def _action(self, action_id: str, incoming: State) -> Outcomes: + action = self.forest.action(action_id) + self._collect_action(action) + node = self.nodes[action_id] + if isinstance(action, Expression): + return self._expression(action_id, node, incoming) + method_name, consumes_node = _ACTION_HANDLERS.get( + type(node), ("_simple_command", True) + ) + handler = getattr(self, method_name) + if consumes_node: + return handler(action_id, node, incoming) + return handler(action_id, incoming) + + def _expression(self, action_id: str, node: ast.AST, incoming: State) -> Outcomes: + if isinstance(node, ast.IfExp): + after_test = self._named_parameter(action_id, "condition", incoming) + branches = tuple( + self._named_parameter(action_id, name, after_test) + for name in ("then", "otherwise") + ) + result = _union_states(branches) + elif isinstance(node, ast.BoolOp): + result = self._lazy_parameters(action_id, incoming) + else: + excluded = ( + frozenset({"body"}) if isinstance(node, ast.Lambda) else frozenset() + ) + result = self._staged_parameters(action_id, incoming, excluded) + return {"normal": self._apply(action_id, SELF, result)} + + def _simple_command( + self, action_id: str, node: ast.AST, incoming: State + ) -> Outcomes: + result = self._staged_parameters(action_id, incoming, frozenset()) + result = self._apply(action_id, SELF, result) + if isinstance(node, ast.Break): + return {"break": result} + if isinstance(node, ast.Continue): + return {"continue": result} + if isinstance(node, ast.Raise): + return {"raise": result} + if isinstance(node, ast.Return): + return {"return": result} + return {"normal": result} + + def _if(self, action_id: str, incoming: State) -> Outcomes: + after_condition = self._named_parameter(action_id, "condition", incoming) + admitted = self._apply(action_id, SELF, after_condition) + body = self._suite(action_id, "body", admitted) + orelse_parameters = self._parameters(action_id, "orelse") + orelse = ( + self._parameter(orelse_parameters[0], admitted) + if orelse_parameters + else {"normal": dict(admitted)} + ) + return _union_outcomes((body, orelse)) + + def _match(self, action_id: str, incoming: State) -> Outcomes: + subject = self._named_parameter(action_id, "subject", incoming) + admitted = self._apply(action_id, SELF, subject) + alternatives: list[Outcomes] = [{"normal": dict(admitted)}] + for prefix in self._case_prefixes(action_id): + selected = self._named_parameter(action_id, f"{prefix}_pattern", admitted) + selected = self._named_parameter(action_id, f"{prefix}_guard", selected) + alternatives.append(self._suite(action_id, f"{prefix}_body", selected)) + return _union_outcomes(tuple(alternatives)) + + def _while(self, action_id: str, incoming: State) -> Outcomes: + head = dict(incoming) + baseline: frozenset[tuple[str, str, Identity]] | None = None + outer_context = self._feedback_context + while True: + self._feedback_context = ( + (action_id, baseline) if baseline is not None else outer_context + ) + condition = self._named_parameter(action_id, "condition", head) + body = self._suite(action_id, "body", condition) + self._feedback_context = outer_context + baseline = frozenset(self.reaching) if baseline is None else baseline + back = tuple(body[kind] for kind in ("normal", "continue") if kind in body) + candidate = _union_states((incoming, *back)) + if candidate == head: + break + head = candidate + condition = self._named_parameter(action_id, "condition", head) + body = self._suite(action_id, "body", condition) + orelse = self._suite(action_id, "orelse", condition) + results: list[Outcomes] = [orelse] + if "break" in body: + results.append({"normal": body["break"]}) + results.append( + {kind: state for kind, state in body.items() if kind in {"raise", "return"}} + ) + return self._finish(action_id, _union_outcomes(tuple(results)), retain=True) + + def _for(self, action_id: str, incoming: State) -> Outcomes: + before = self._named_parameter(action_id, "iterable", incoming) + target = self._parameters(action_id, "target")[0] + head = dict(before) + baseline: frozenset[tuple[str, str, Identity]] | None = None + outer_context = self._feedback_context + while True: + self._feedback_context = ( + (action_id, baseline) if baseline is not None else outer_context + ) + iteration = self._normal(self._parameter(target, head)) + body = self._suite(action_id, "body", iteration) + self._feedback_context = outer_context + baseline = frozenset(self.reaching) if baseline is None else baseline + back = tuple(body[kind] for kind in ("normal", "continue") if kind in body) + candidate = _union_states((before, *back)) + if candidate == head: + break + head = candidate + iteration = self._normal(self._parameter(target, head)) + body = self._suite(action_id, "body", iteration) + orelse = self._suite(action_id, "orelse", head) + results: list[Outcomes] = [orelse] + if "break" in body: + results.append({"normal": body["break"]}) + results.append( + {kind: state for kind, state in body.items() if kind in {"raise", "return"}} + ) + return self._finish(action_id, _union_outcomes(tuple(results)), retain=True) + + def _try(self, action_id: str, incoming: State) -> Outcomes: + admitted = self._apply(action_id, SELF, incoming) + body = self._suite(action_id, "body", admitted) + exception_seed = _union_states((admitted, *body.values())) + handlers = tuple( + self._handler(action_id, prefix, exception_seed) + for prefix in self._handler_prefixes(action_id) + ) + normal_body = body.get("normal") + orelse = ( + self._suite(action_id, "orelse", normal_body) + if normal_body is not None + else {} + ) + propagated = {kind: state for kind, state in body.items() if kind != "normal"} + combined = _union_outcomes((orelse, propagated, *handlers)) + finalbody = self._parameters(action_id, "finalbody") + if finalbody: + combined = self._apply_finalbody(combined, finalbody[0]) + return combined + + def _handler(self, action_id: str, prefix: str, incoming: State) -> Outcomes: + selected = self._named_parameter(action_id, f"{prefix}_type", incoming) + targets = self._parameters(action_id, f"{prefix}_target") + target = targets[0] if targets else None + if target is not None: + selected = self._normal(self._parameter(target, selected, defer_exit=True)) + result = self._suite(action_id, f"{prefix}_body", selected) + if target is None: + return result + return { + kind: self._apply(action_id, (target.id, EXIT), state) + for kind, state in result.items() + } + + def _with(self, action_id: str, incoming: State) -> Outcomes: + result = dict(incoming) + parameters = self._ordered_parameters(action_id) + body: Parameter | None = None + for parameter in parameters: + if parameter.role is ParameterRole.SUITE: + body = parameter + continue + result = self._normal(self._parameter(parameter, result)) + result = self._apply(action_id, SELF, result) + outcomes = ( + self._parameter(body, result) if body is not None else {"normal": result} + ) + return outcomes + + def _definition(self, action_id: str, node: ast.AST, incoming: State) -> Outcomes: + result = dict(incoming) + for parameter in self._ordered_parameters(action_id): + if parameter.id in self.dormant: + continue + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and parameter.role is ParameterRole.SUITE + ): + continue + result = self._normal(self._parameter(parameter, result)) + return {"normal": self._apply(action_id, SELF, result)} + + def _staged_parameters( + self, action_id: str, incoming: State, excluded: frozenset[str] + ) -> State: + result = dict(incoming) + for parameter in self._ordered_parameters(action_id): + if parameter.id in self.dormant or parameter.name in excluded: + continue + if parameter.role is ParameterRole.SUITE: + continue + result = self._normal(self._parameter(parameter, result)) + return result + + def _lazy_parameters(self, action_id: str, incoming: State) -> State: + result = dict(incoming) + exits: list[State] = [] + for parameter in self._ordered_parameters(action_id): + if parameter.role is not ParameterRole.VALUE: + continue + result = self._normal(self._parameter(parameter, result)) + exits.append(result) + return _union_states(tuple(exits)) if exits else result + + def _named_parameter(self, action_id: str, name: str, incoming: State) -> State: + parameters = self._parameters(action_id, name) + return ( + self._normal(self._parameter(parameters[0], incoming)) + if parameters + else dict(incoming) + ) + + def _suite(self, action_id: str, name: str, incoming: State) -> Outcomes: + parameters = self._parameters(action_id, name) + return ( + self._parameter(parameters[0], incoming) + if parameters + else {"normal": dict(incoming)} + ) + + def _parameter( + self, parameter: Parameter, incoming: State, defer_exit: bool = False + ) -> Outcomes: + entered = self._apply(parameter.owner, (parameter.id, ENTER), incoming) + outcomes = self._sequence(parameter.actions, entered) + if defer_exit: + return outcomes + return { + kind: self._apply(parameter.owner, (parameter.id, EXIT), state) + for kind, state in outcomes.items() + } + + def _apply( + self, action_id: str, anchor: Anchor, incoming: State, retain: bool = False + ) -> State: + uses, defines, kills = self.events.get((action_id, anchor), ((), (), ())) + for identity in uses: + self._record_reaching(action_id, identity, incoming) + result = dict(incoming) + for identity in kills: + result.pop(identity, None) + for identity in defines: + previous = result.get(identity) if retain else None + producers = frozenset() if previous is None else previous.producers + result[identity] = _DefinitionState( + producers | frozenset({action_id}), False + ) + return result + + def _record_reaching( + self, action_id: str, identity: Identity, incoming: State + ) -> None: + state = incoming.get(identity) + if state is None or state.may_be_unbound: + self.unbound_uses.add(identity) + self.unbound_occurrences.add((action_id, identity)) + for producer in () if state is None else state.producers: + if producer != action_id: + row = producer, action_id, identity + self.reaching.add(row) + self._record_iteration_feedback(row) + + def _record_iteration_feedback(self, row: tuple[str, str, Identity]) -> None: + if self._feedback_context is None: + return + owner, baseline = self._feedback_context + if row not in baseline: + self.iteration_feedback.add((owner, *row)) + + def _finish(self, action_id: str, outcomes: Outcomes, retain: bool) -> Outcomes: + return { + kind: self._apply(action_id, SELF, state, retain) + for kind, state in outcomes.items() + } + + def _apply_finalbody(self, outcomes: Outcomes, parameter: Parameter) -> Outcomes: + results: list[Outcomes] = [] + for original_kind, state in outcomes.items(): + final = self._parameter(parameter, state) + normal = final.pop("normal", None) + if normal is not None: + results.append({original_kind: normal}) + results.append(final) + return _union_outcomes(tuple(results)) + + def _handler_prefixes(self, action_id: str) -> tuple[str, ...]: + return tuple( + parameter.name.removesuffix("_body") + for parameter in self._ordered_parameters(action_id) + if parameter.role is ParameterRole.SUITE + and ( + parameter.name == "handler_body" + or parameter.name.startswith("handler_") + and parameter.name.endswith("_body") + ) + ) + + def _case_prefixes(self, action_id: str) -> tuple[str, ...]: + return tuple( + parameter.name.removesuffix("_body") + for parameter in self._ordered_parameters(action_id) + if parameter.role is ParameterRole.SUITE + and parameter.name.startswith("case") + and parameter.name.endswith("_body") + ) + + def _ordered_parameters(self, action_id: str) -> tuple[Parameter, ...]: + action = self.forest.action(action_id) + indexed = tuple( + (index, self.forest.parameter(parameter_id)) + for index, parameter_id in enumerate(action.parameters) + ) + return tuple( + parameter + for _, parameter in sorted( + indexed, key=lambda item: (self.indices[item[1].id], item[0]) + ) + ) + + def _parameters(self, action_id: str, name: str) -> tuple[Parameter, ...]: + return tuple( + parameter + for parameter in self._ordered_parameters(action_id) + if parameter.name == name + ) + + @staticmethod + def _normal(outcomes: Outcomes) -> State: + if set(outcomes) != {"normal"}: + raise ForestBuildError("a parameter completed abruptly") + return outcomes["normal"] + + +def _union_outcomes(outcomes: tuple[Outcomes, ...]) -> Outcomes: + kinds = {kind for outcome in outcomes for kind in outcome} + return { + kind: _union_states( + tuple(outcome[kind] for outcome in outcomes if kind in outcome) + ) + for kind in kinds + } + + +def _union_states(states: tuple[State, ...]) -> State: + """Merge path states at a join, preserving every alternative producer. + + Producer sets are immutable, so a key carried by exactly one incoming state + reuses that state's set instead of being rebuilt. Rebuilding every set + element by element dominated analysis on real programs and made it + superlinear in program size, which §13.19 forbids. + """ + + if len(states) == 1: + return dict(states[0]) + merged: State = {} + counts: _DefinitionCounts = {} + for state in states: + _merge_state_into(merged, counts, state) + return _mark_missing_definitions(merged, counts, len(states)) + + +def _merge_state_into(merged: State, counts: _DefinitionCounts, state: State) -> None: + """Fold one path state into the accumulator, reusing its immutable sets.""" + + for key, producers in state.items(): + existing = merged.get(key) + counts[key] = counts.get(key, 0) + 1 + merged[key] = ( + producers + if existing is None or existing is producers + else _DefinitionState( + existing.producers | producers.producers, + existing.may_be_unbound or producers.may_be_unbound, + ) + ) + + +def _mark_missing_definitions( + merged: State, counts: _DefinitionCounts, path_count: int +) -> State: + """Mark identities absent from at least one legal incoming path.""" + + return { + identity: ( + row + if counts[identity] == path_count or row.may_be_unbound + else _DefinitionState(row.producers, True) + ) + for identity, row in merged.items() + } + + +@dataclass(frozen=True) +class _DemandRule: + field: str + first_index: int = 0 + + def matches(self, field: str, item_index: int) -> bool: + """Return whether one component is selected by this demand rule.""" + + return self.field == field and item_index >= self.first_index + + +type _StageSelector = str | tuple[str, ...] | frozenset[str] + + +@dataclass(frozen=True) +class _EvaluationProtocol: + stages: tuple[_StageSelector, ...] = () + demands: tuple[_DemandRule, ...] = () + orderer: Callable[[tuple[_Component, ...]], tuple[int, ...]] | None = None + refiner: Callable[[_Component, tuple[_Component, ...]], bool] | None = None + + def evaluation_indices(self, components: tuple[_Component, ...]) -> tuple[int, ...]: + """Return the protocol's deterministic stage index per component.""" + + return ( + self.orderer(components) + if self.orderer + else _staged_indices(components, self.stages) + ) + + def is_demanded( + self, component: _Component, components: tuple[_Component, ...] + ) -> bool: + """Return whether evaluating this component may require suspension.""" + + base = any( + rule.matches(component.origin_field, component.origin_index) + for rule in self.demands + ) + return self.refiner(component, components) if self.refiner is not None else base + + +def _protocol_metadata( + forest: ActionForest, +) -> tuple[frozenset[str], frozenset[str], dict[str, int]]: + return _metadata(_validated_builder(forest)) + + +def _validated_builder(forest: ActionForest) -> _Builder: + """Return the forest-owned source-exact structural index.""" + + return forest._source_builder + + +def _metadata( + builder: _Builder, +) -> tuple[frozenset[str], frozenset[str], dict[str, int]]: + demanded: set[str] = set() + dormant: set[str] = set() + indices: dict[str, int] = {} + for action in builder.actions: + node = builder.action_nodes[action.id] + components = builder.components_by_action[action.id] + protocol = _PROTOCOLS.get(type(node), _DEFAULT_PROTOCOL) + action_indices = protocol.evaluation_indices(components) + for parameter_id, component, index in zip( + action.parameters, components, action_indices, strict=True + ): + indices[parameter_id] = index + if protocol.is_demanded(component, components): + demanded.add(parameter_id) + if component.origin_field in _DORMANT_FIELDS.get(type(node), frozenset()): + dormant.add(parameter_id) + return frozenset(demanded), frozenset(dormant), indices + + +def _staged_indices( + components: tuple[_Component, ...], plan: tuple[_StageSelector, ...] +) -> tuple[int, ...]: + if not plan: + return tuple(range(len(components))) + remaining = list(range(len(components))) + result: list[int | None] = [None] * len(components) + stage = 0 + for item in plan: + stage = _assign_stage(components, item, remaining, result, stage) + for index in remaining: + result[index] = stage + stage += 1 + return tuple(int(item) for item in result) + + +def _assign_stage( + components: tuple[_Component, ...], + item: _StageSelector, + remaining: list[int], + result: list[int | None], + stage: int, +) -> int: + selectors = (item,) if isinstance(item, str) else item + matches = [ + index + for index in remaining + if any(_matches(components[index].name, selector) for selector in selectors) + ] + if isinstance(item, frozenset): + for index in matches: + result[index] = stage + remaining.remove(index) + return stage + bool(matches) + for index in matches: + result[index] = stage + stage += 1 + remaining.remove(index) + return stage + + +def _comprehension_indices(components: tuple[_Component, ...]) -> tuple[int, ...]: + prefixes = sorted( + {prefix for item in components if (prefix := _generator_prefix(item.name))}, + key=lambda prefix: min( + _component_position(item) + for item in components + if _generator_prefix(item.name) == prefix + ), + ) + order: list[int] = [] + for prefix in prefixes: + members = [ + index + for index, item in enumerate(components) + if _generator_prefix(item.name) == prefix + ] + for suffix in ("_iter", "_target", "_ifs"): + order.extend( + index + for index in members + if suffix in components[index].name and index not in order + ) + order.extend(index for index in range(len(components)) if index not in order) + result = [0] * len(components) + for stage, index in enumerate(order): + result[index] = stage + return tuple(result) + + +def _generator_prefix(name: str) -> str | None: + for marker in ("_target", "_iter", "_ifs"): + if marker in name and name.startswith("generator"): + return name.split(marker, maxsplit=1)[0] + return None + + +def _matches(name: str, selector: str) -> bool: + return name == selector or name.startswith(selector + "_") + + +def _comprehension_demand( + component: _Component, components: tuple[_Component, ...] +) -> bool: + prefix = _generator_prefix(component.name) + if prefix is None: + return True + prefixes = list( + dict.fromkeys( + item + for item in (_generator_prefix(candidate.name) for candidate in components) + if item is not None + ) + ) + return not (prefixes and prefix == prefixes[0] and "_iter" in component.name) + + +_ACTION_HANDLERS: dict[type[ast.AST], tuple[str, bool]] = { + ast.If: ("_if", False), + ast.Match: ("_match", False), + ast.While: ("_while", False), + ast.For: ("_for", False), + ast.Try: ("_try", False), + ast.With: ("_with", False), + ast.FunctionDef: ("_definition", True), + ast.AsyncFunctionDef: ("_definition", True), + ast.ClassDef: ("_definition", True), +} +_DORMANT_FIELDS: dict[type[ast.AST], frozenset[str]] = { + ast.FunctionDef: frozenset({"body"}), + ast.Lambda: frozenset({"body"}), +} +_DEFAULT_PROTOCOL = _EvaluationProtocol() +_COMPREHENSION_PROTOCOL = _EvaluationProtocol( + demands=(_DemandRule("generators"),), + orderer=_comprehension_indices, + refiner=_comprehension_demand, +) +_PROTOCOLS: dict[type[ast.AST], _EvaluationProtocol] = { + ast.FunctionDef: _EvaluationProtocol( + stages=( + "decorator", + "signature_default", + "signature_annotation", + "return_annotation", + "target", + "body", + ) + ), + ast.ClassDef: _EvaluationProtocol( + stages=("decorator", ("base", "keyword"), "body", "target") + ), + ast.If: _EvaluationProtocol( + stages=("condition", frozenset({"body", "orelse"})), + demands=(_DemandRule("body"), _DemandRule("orelse")), + ), + ast.While: _EvaluationProtocol( + stages=("condition", "body", "orelse"), + demands=(_DemandRule("test"), _DemandRule("body"), _DemandRule("orelse")), + ), + ast.For: _EvaluationProtocol( + stages=("iterable", "target", "body", "orelse"), + demands=(_DemandRule("body"), _DemandRule("orelse")), + ), + ast.Try: _EvaluationProtocol( + stages=("body", frozenset({"handler", "orelse"}), "finalbody"), + demands=( + _DemandRule("body"), + _DemandRule("handlers"), + _DemandRule("orelse"), + _DemandRule("finalbody"), + ), + ), + ast.With: _EvaluationProtocol( + stages=("item", "body"), demands=(_DemandRule("body"),) + ), + ast.Assign: _EvaluationProtocol(stages=("value", "target")), + ast.AnnAssign: _EvaluationProtocol(stages=("value", "target", "annotation")), + ast.NamedExpr: _EvaluationProtocol(stages=("value", "target")), + ast.IfExp: _EvaluationProtocol( + stages=("condition", frozenset({"then", "otherwise"})), + demands=(_DemandRule("body"), _DemandRule("orelse")), + ), + ast.Assert: _EvaluationProtocol(demands=(_DemandRule("msg"),)), + ast.BoolOp: _EvaluationProtocol(demands=(_DemandRule("values", 1),)), + ast.Lambda: _EvaluationProtocol(demands=(_DemandRule("body"),)), + ast.Compare: _EvaluationProtocol(demands=(_DemandRule("comparators", 1),)), + ast.ListComp: _COMPREHENSION_PROTOCOL, + ast.SetComp: _COMPREHENSION_PROTOCOL, + ast.DictComp: _COMPREHENSION_PROTOCOL, + ast.GeneratorExp: _COMPREHENSION_PROTOCOL, +} diff --git a/py2udf/src/main/python/python_to_workflow/mosaic/analysis/scopes.py b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/scopes.py new file mode 100644 index 00000000000..8cd2cdb2e9b --- /dev/null +++ b/py2udf/src/main/python/python_to_workflow/mosaic/analysis/scopes.py @@ -0,0 +1,286 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Source-exact lexical scope identities for modular Mosaic analysis. + +The index is an ephemeral view of the ActionForest and CPython's symbol-table +classification. It is neither a provider fact nor a second source tree. Its +named records keep scope meaning out of anonymous dict/tuple positions. +""" + +from __future__ import annotations + +import ast +import symtable +from collections.abc import Mapping +from dataclasses import dataclass +from functools import total_ordering +from types import MappingProxyType +from typing import Literal + +from python_to_workflow.mosaic.forest import ( + ActionForest, + ForestBuildError, + Parameter, + ParameterRole, + _Builder, +) + +type ActionId = str +type ActionIds = tuple[ActionId, ...] +type SourceSignature = str +type SymbolTable = symtable.SymbolTable + + +@total_ordering +@dataclass(frozen=True) +class ScopeId: + """Stable semantic identity of one real source callable/class namespace.""" + + path: str + signature: SourceSignature + captures: tuple[str, ...] + + def __lt__(self, other: object) -> bool: + """Provide one deterministic order for canonical mixed-scope rows.""" + + if isinstance(other, str): + return False + if isinstance(other, ScopeId): + return (self.path, self.signature, self.captures) < ( + other.path, + other.signature, + other.captures, + ) + return NotImplemented + + +type ModuleScope = Literal["module"] +type LexicalScopeId = ModuleScope | ScopeId +type BindingIdentity = tuple[LexicalScopeId, str] + + +@dataclass(frozen=True) +class LexicalScope: + """One real source namespace and its directly executed forest Actions.""" + + parent: LexicalScopeId | None + roots: ActionIds + actions: frozenset[ActionId] + symbols: SymbolTable + + +type ScopeTable = Mapping[LexicalScopeId, LexicalScope] +type ActionScopeTable = Mapping[ActionId, LexicalScopeId] + + +@dataclass(frozen=True) +class LexicalScopeIndex: + """Immutable joins from source Actions and definitions to lexical scopes.""" + + builder: _Builder + scopes: ScopeTable + action_scopes: ActionScopeTable + owner_scopes: ActionScopeTable + + +@dataclass +class _ScopeDraft: + """Transient construction state; frozen before it leaves the indexer.""" + + parent: LexicalScopeId | None + roots: ActionIds + actions: set[ActionId] + symbols: SymbolTable + + +class _ScopeIndexer: + """Build the lexical view once from the source-exact forest.""" + + def __init__(self, forest: ActionForest) -> None: + self.forest = forest + self.builder = forest._source_builder + module = symtable.symtable(forest.source, "", "exec") + self.scopes: dict[LexicalScopeId, _ScopeDraft] = { + "module": _ScopeDraft(None, forest.roots, set(), module) + } + self.action_scopes: dict[ActionId, LexicalScopeId] = {} + self.owner_scopes: dict[ActionId, LexicalScopeId] = {} + + def build(self) -> LexicalScopeIndex: + """Assign every Action exactly once and freeze every index view.""" + + self._assign_actions(self.forest.roots, "module") + scopes = { + scope_id: LexicalScope( + draft.parent, + draft.roots, + frozenset(draft.actions), + draft.symbols, + ) + for scope_id, draft in self.scopes.items() + } + return LexicalScopeIndex( + self.builder, + MappingProxyType(scopes), + MappingProxyType(dict(self.action_scopes)), + MappingProxyType(dict(self.owner_scopes)), + ) + + def _assign_actions(self, action_ids: ActionIds, scope_id: LexicalScopeId) -> None: + """Assign a source-ordered Action sequence to one execution scope.""" + + for action_id in action_ids: + self._assign_action(action_id, scope_id) + + def _assign_action(self, action_id: ActionId, scope_id: LexicalScopeId) -> None: + """Assign an Action and recursively divert only real body scopes.""" + + self.scopes[scope_id].actions.add(action_id) + self.action_scopes[action_id] = scope_id + node = self.builder.action_nodes[action_id] + child_scope = self._definition_scope(action_id, node, scope_id) + for parameter_id in self.forest.action(action_id).parameters: + parameter = self.forest.parameter(parameter_id) + destination = _body_destination(parameter, child_scope, scope_id) + if destination == child_scope: + self.scopes[child_scope].roots = parameter.actions + self._assign_actions(parameter.actions, destination) + + def _definition_scope( + self, owner: ActionId, node: ast.AST, parent_scope: LexicalScopeId + ) -> ScopeId | None: + """Register a function/class body scope; headers remain in the parent.""" + + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return None + table = self._matching_table(parent_scope, node) + parent_path = _scope_path(parent_scope) + path = node.name if parent_path == "module" else f"{parent_path}.{node.name}" + captures = tuple( + sorted( + symbol.get_name() + for symbol in table.get_symbols() + if symbol.is_free() or symbol.is_nonlocal() + ) + ) + scope_id = ScopeId(path, _source_signature(node), captures) + self.scopes[scope_id] = _ScopeDraft(parent_scope, (), set(), table) + self.owner_scopes[owner] = scope_id + return scope_id + + def _matching_table( + self, + parent_scope: LexicalScopeId, + node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, + ) -> SymbolTable: + """Join one forest definition to CPython's corresponding symbol table.""" + + kind = "class" if isinstance(node, ast.ClassDef) else "function" + used = {id(scope.symbols) for scope in self.scopes.values()} + candidates = tuple( + table + for table in self.scopes[parent_scope].symbols.get_children() + if table.get_name() == node.name + and table.get_lineno() == node.lineno + and table.get_type() == kind + and id(table) not in used + ) + if len(candidates) != 1: + raise ForestBuildError( + f"cannot identify lexical scope for {node.name!r} at line {node.lineno}" + ) + return candidates[0] + + +def build_lexical_scope_index(forest: ActionForest) -> LexicalScopeIndex: + """Return the lexical view for the source forest.""" + + return _ScopeIndexer(forest).build() + + +def binding_identity( + index: LexicalScopeIndex, scope_id: LexicalScopeId, name: str +) -> BindingIdentity: + """Name a binding according to CPython's lexical symbol classification.""" + + if scope_id == "module": + return "module", name + try: + symbol = index.scopes[scope_id].symbols.lookup(name) + except KeyError: + return "module", name + if symbol.is_global(): + return "module", name + if symbol.is_free() or symbol.is_nonlocal(): + return _free_binding_scope(index, scope_id, name), name + return scope_id, name + + +def _free_binding_scope( + index: LexicalScopeIndex, scope_id: LexicalScopeId, name: str +) -> LexicalScopeId: + """Find the real defining namespace of one free/nonlocal cell.""" + + current = index.scopes[scope_id].parent + while current not in {None, "module"}: + try: + symbol = index.scopes[current].symbols.lookup(name) + except KeyError: + current = index.scopes[current].parent + continue + if symbol.is_local() or symbol.is_parameter(): + return current + current = index.scopes[current].parent + return "module" + + +def _body_destination( + parameter: Parameter, + child_scope: ScopeId | None, + parent_scope: LexicalScopeId, +) -> LexicalScopeId: + """Keep headers in the parent and move only the definition body.""" + + is_body = ( + child_scope is not None + and parameter.role is ParameterRole.SUITE + and parameter.name == "body" + ) + return child_scope if is_body else parent_scope + + +def _scope_path(scope_id: LexicalScopeId) -> str: + """Project the readable lexical path without parsing semantic behavior.""" + + return scope_id if isinstance(scope_id, str) else scope_id.path + + +def _source_signature( + node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, +) -> SourceSignature: + """Produce a deterministic signature component from source AST fields.""" + + if isinstance(node, ast.ClassDef): + bases = ", ".join(ast.unparse(base) for base in node.bases) + keywords = ", ".join( + f"{keyword.arg}={ast.unparse(keyword.value)}" for keyword in node.keywords + ) + separator = ", " if bases and keywords else "" + return f"({bases}{separator}{keywords})" + parameters = ast.unparse(node.args) + returns = "" if node.returns is None else f" -> {ast.unparse(node.returns)}" + return f"({parameters}){returns}" diff --git a/py2udf/src/test/python/python_to_workflow/mosaic/test_controlwalk.py b/py2udf/src/test/python/python_to_workflow/mosaic/test_controlwalk.py new file mode 100644 index 00000000000..3e214ecf42c --- /dev/null +++ b/py2udf/src/test/python/python_to_workflow/mosaic/test_controlwalk.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for control-flow traversal.""" + +import ast + +from python_to_workflow.mosaic.analysis.controlwalk import ( + SELF, + compile_controlwalk, + controlwalk, + controlwalk_region_result, +) +from python_to_workflow.mosaic.analysis.scopes import build_lexical_scope_index +from python_to_workflow.mosaic.forest import build_forest + + +def _section(payload, name): + return dict(payload)[name] + + +def test_unreachable_definition_is_in_inventory_but_not_reaching() -> None: + """Inventory and executable paths must remain separate authorities.""" + + forest = build_forest( + "try:\n" + " raise RuntimeError()\n" + " value = 1\n" + "except RuntimeError:\n" + " print(value)\n" + ) + + def events(action): + if action.id == "s1.body.1": + return ((SELF, (), (("module", "value"),), ()),) + if action.id == "s1.handler_body.0.positional_0.0": + return ((SELF, (("module", "value"),), (), ()),) + return ((SELF, (), (), ()),) + + payload = controlwalk(forest, events) + + assert ("s1.body.1", ("module", "value")) in _section(payload, "definitions") + assert not any( + producer == "s1.body.1" and identity == ("module", "value") + for producer, _consumer, identity in _section(payload, "reaching") + ) + + +def test_region_inventory_excludes_deferred_lambda_body() -> None: + """A containing lexical query must not inspect deferred lambda code.""" + + forest = build_forest( + "outside = 1\n" + "def target(value):\n" + " delayed = lambda hidden: hidden + outside\n" + " return delayed\n" + "result = target(outside)\n" + ) + scopes = build_lexical_scope_index(forest) + owner = next( + action_id + for action_id, node in forest._source_builder.action_nodes.items() + if isinstance(node, ast.FunctionDef) and node.name == "target" + ) + lambda_owner = next( + action_id + for action_id, node in forest._source_builder.action_nodes.items() + if isinstance(node, ast.Lambda) + ) + body = next( + forest.parameter(parameter_id) + for parameter_id in forest.action(lambda_owner).parameters + if forest.parameter(parameter_id).name == "body" + ) + deferred = frozenset(body.actions) + scope = scopes.scopes[scopes.owner_scopes[owner]] + visited = set() + + def events(action): + visited.add(action.id) + return ((SELF, (), (), ()),) + + controlwalk_region_result( + forest, + scope.roots, + {}, + events, + compile_controlwalk(forest), + ) + + assert deferred + assert deferred.isdisjoint(visited)