From 56bb8f192aa698b38b3480cfe9264f7f0fa3b23c Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Fri, 31 Jul 2026 08:20:33 +0200 Subject: [PATCH] refactor(language): the front end is a package, and the fence is a test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ten modules from the file reader to the dim checker never reach forward to a consumer — no `lowering`, no `piecewise`, no `sources`, no `api`, no `relational`. That was true by habit and unenforced: hard rule 0 claims "the layers are ordered, and imports prove it", but only the lazy-import half was ever checked, so nothing stopped `expression_parser.py` from importing `lowering.py` next month. Move nine of them under `src/lpspec/language/` and enforce membership off the path, the way `relational/`, `linopy/` and `typeset/` already are. It is the mirror of hard rule 2: the engine may not see the schema or the AST, and now the language may not see what is done with the AST it produces. That is what keeps `lps.check()` a pass with no data and no plan, and a second consumer cheap rather than a second opinion. `errors.py` stays at the top level on purpose — it is the dependency-free leaf *both* fences point at (`ENGINE_MAY_IMPORT`), and moving it in would put the language's import path on the engine's list. `lowering.py`, `piecewise.py`, `sources.py` and `api.py` stay too: each is legitimately on the line between the halves, and the flat namespace is where that belongs. No behaviour change — imports and paths only. Co-Authored-By: Claude Opus 5 (1M context) --- docs/ARCHITECTURE.md | 57 ++++++++++++------- examples/walkthrough.py | 4 +- pyproject.toml | 2 +- src/lpspec/__init__.py | 2 +- src/lpspec/api.py | 6 +- src/lpspec/language/__init__.py | 21 +++++++ src/lpspec/{ => language}/_yaml.py | 0 src/lpspec/{ => language}/dimensions.py | 8 +-- src/lpspec/{ => language}/expansion.py | 4 +- .../{ => language}/expression_parser.py | 0 src/lpspec/{ => language}/helpers.py | 0 src/lpspec/{ => language}/resolution.py | 10 ++-- src/lpspec/{ => language}/schema.py | 2 +- src/lpspec/{ => language}/validation.py | 14 ++--- src/lpspec/{ => language}/where_parser.py | 0 src/lpspec/linopy/__init__.py | 6 +- src/lpspec/linopy/builder.py | 14 ++--- src/lpspec/linopy/loader.py | 4 +- src/lpspec/lowering.py | 14 ++--- src/lpspec/piecewise.py | 8 +-- src/lpspec/sources.py | 2 +- src/lpspec/typeset/__init__.py | 4 +- src/lpspec/typeset/symbols.py | 4 +- src/lpspec/typeset/walk.py | 14 ++--- tests/conftest.py | 6 +- tests/differential.py | 2 +- tests/test_api.py | 2 +- tests/test_architecture.py | 52 ++++++++++++++++- tests/test_dimensions.py | 6 +- tests/test_doc_examples.py | 2 +- tests/test_expansion.py | 8 +-- tests/test_linopy_lane.py | 4 +- tests/test_lowering.py | 4 +- tests/test_parser.py | 4 +- tests/test_relational.py | 2 +- tests/test_resolution.py | 10 ++-- tests/test_resolution_parity.py | 4 +- tests/test_schema.py | 2 +- tests/test_validation.py | 4 +- tests/test_yaml_loading.py | 4 +- tools/gallery_math.py | 2 +- 41 files changed, 200 insertions(+), 118 deletions(-) create mode 100644 src/lpspec/language/__init__.py rename src/lpspec/{ => language}/_yaml.py (100%) rename src/lpspec/{ => language}/dimensions.py (98%) rename src/lpspec/{ => language}/expansion.py (98%) rename src/lpspec/{ => language}/expression_parser.py (100%) rename src/lpspec/{ => language}/helpers.py (100%) rename src/lpspec/{ => language}/resolution.py (98%) rename src/lpspec/{ => language}/schema.py (99%) rename src/lpspec/{ => language}/validation.py (96%) rename src/lpspec/{ => language}/where_parser.py (100%) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index feb6dc93..8547d514 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -25,8 +25,8 @@ below protects it. ```mermaid flowchart TB - Y[YAML file] -->|"parse + validate
(schema.py, validation.py)"| MS[MathSchema] - MS -->|"expand macros: / expressions: (expansion.py)
expand piecewise: blocks (piecewise.py)
resolve names to typed nodes (resolution.py)
check dim sets (dimensions.py)
— backends never see any of them"| AST["core AST
= the only contract between layers
fully typed: names resolved, dims checked"] + Y[YAML file] -->|"parse + validate
(language/schema.py, language/validation.py)"| MS[MathSchema] + MS -->|"expand macros: / expressions: (language/expansion.py)
expand piecewise: blocks (piecewise.py)
resolve names to typed nodes (language/resolution.py)
check dim sets (language/dimensions.py)
— backends never see any of them"| AST["core AST
= the only contract between layers
fully typed: names resolved, dims checked"] AST -->|"api.py: check / build / solve / write"| LOWER AST -.->|"lpspec.linopy
(opt-in shim: build / extend)"| BUILD LOWER -->|"outside the language:
LanguageError naming the construct"| ERR["load error
(no fallback)"] @@ -171,7 +171,12 @@ that made an implementation choice load-bearing in the language's rulebook. and the plan/query/xarray are backend-private. The AST crossing that seam is **fully resolved** — names are typed `Variable`/`Parameter`/`Dimension` nodes — so a backend cannot hold its own opinion about what a name refers to. Resolving independently is - how the two lanes silently disagreed about scoping before. + how the two lanes silently disagreed about scoping before. The waist is + closed from the front too: nothing under `src/lpspec/language/` imports + `lowering`, `piecewise`, `sources`, `api` or any consuming subpackage, so + what a model *means* cannot depend on what is done with it. That is the + mirror of rule 2 and it is enforced the same way, off the path + (`LANGUAGE_MAY_IMPORT`). 2. **The engine knows nothing about linopy, xarray or YAML.** `src/lpspec/relational/` goes polars → highspy → solver, with linopy's semantics as a spec to match rather than code to share; it never sees the @@ -271,21 +276,21 @@ than discovered at solve time. | Module | Role | |---|---| -| `_yaml.py` | the only place a file is read: YAML 1.2 booleans, duplicate keys refused | -| `schema.py` | pydantic schema incl. `expressions:` / `macros:` / `piecewise:` | -| `expression_parser.py`, `where_parser.py` | text → core AST; grammar only, dependency-free | -| `expansion.py` | named-expression / macro substitution (pre-dispatch) | -| `resolution.py` | one flat namespace; `NameNode` → typed `Variable`/`Parameter`/`Dimension` nodes | -| `dimensions.py` | static dim-set checking over the resolved AST | -| `validation.py` | load-time: parse, expand, resolve, check everything | +| `language/_yaml.py` | the only place a file is read: YAML 1.2 booleans, duplicate keys refused | +| `language/schema.py` | pydantic schema incl. `expressions:` / `macros:` / `piecewise:` | +| `language/expression_parser.py`, `language/where_parser.py` | text → core AST; grammar only, dependency-free | +| `language/expansion.py` | named-expression / macro substitution (pre-dispatch) | +| `language/resolution.py` | one flat namespace; `NameNode` → typed `Variable`/`Parameter`/`Dimension` nodes | +| `language/dimensions.py` | static dim-set checking over the resolved AST | +| `language/helpers.py` | the closed set of built-in operators: their *names* and *call shapes* — no registry | +| `language/validation.py` | load-time: parse, expand, resolve, check everything | | `piecewise.py` | `piecewise:` → λ-formulation declarations + curvature guard | | `api.py` | native entry point: `check` / `build` / `solve` / `write`, linopy-free | | `typeset/` | **spike** — resolved AST → LaTeX / Typst / Markdown. A reader, not a lane: no model, no data, no plan ([README](https://github.com/FBumann/lpspec/blob/main/src/lpspec/typeset/README.md)) | | `__main__.py` | `python -m lpspec ` — a shell front for the verbs that bind no data | | `sources.py` | bind runtime data (parquet paths / in-memory tables) to a validated schema | | `lowering.py` | core AST → logical plan (defines the relational subset) | -| `helpers.py` | the closed set of built-in operators: their *names* and *call shapes* — no registry | -| `errors.py` | the exception hierarchy; the one module the engine may import | +| `errors.py` | the exception hierarchy; the one module either fenced side may import | | `relational/plan.py` | frozen logical-plan dataclasses | | `relational/frames.py` | the boundary — caller tables in, via the Arrow PyCapsule protocol | | `relational/compiler.py` | plan → lazy frames; pure, reads nothing | @@ -300,13 +305,23 @@ than discovered at solve time. | `linopy/builder.py` | eager backend: core AST → `linopy.Model` | | `linopy/semantics.py` | where this lane answers linopy's v1 arithmetic convention — one home, as linopy's own `semantics.py` is | -Two subpackages, and the directory *is* the rule in both cases. Everything -under `relational/` is the engine and imports nothing else from the package; -everything under `linopy/` is the opt-in eager lane and is the only code -allowed to import linopy or xarray; everything under `typeset/` reads the AST -and writes text, and reaches neither the plan nor any data. `tests/test_architecture.py` reads -membership off the path, so neither fence can be stepped over by naming a -file differently. +**Four subpackages, and the directory *is* the rule in every case.** Everything +under `language/` produces the AST and may not reach a consumer of it; +everything under `relational/` is the engine and imports nothing else from the +package; everything under `linopy/` is the opt-in eager lane and is the only +code allowed to import linopy or xarray; everything under `typeset/` reads the +AST and writes text, and reaches neither the plan nor any data. +`tests/test_architecture.py` reads membership off the path, so no fence can be +stepped over by naming a file differently. + +`language/` and `relational/` are the two halves of the waist and their fences +point the same way — outward, at `errors.py`, the one leaf both may import. The +modules left at the top level are the ones that are legitimately *both*: +`lowering.py` reads the AST and writes the plan, `piecewise.py` emits +declarations but consults the subset test to do it, `sources.py` binds data to +a validated schema, and `api.py` runs the lot. A module that belongs to neither +side is a module that has to sit on the line, and the flat namespace is where +it sits. ### Naming across the layers @@ -316,7 +331,7 @@ suffix**, which is what keeps the three vocabularies from colliding: | Layer | Suffix | Example | |---|---|---| -| YAML block (`schema.py`) | `Block` | `VariableBlock`, `PiecewiseBlock` | +| YAML block (`language/schema.py`) | `Block` | `VariableBlock`, `PiecewiseBlock` | | Core AST (`*_parser.py`) | `Node` | `VariableNode`, `DimensionComparisonNode` | | Logical plan (`relational/plan.py`) | none / `Declaration` | `Variable`, `VariableDeclaration` | @@ -364,7 +379,7 @@ lowering case → differential test on both sinks → SPEC §5/§7, and this fil structural. Two things are deliberately *not* per-primitive work, because they are one -implementation each: a primitive's dim rule lives only in `dimensions.py` — +implementation each: a primitive's dim rule lives only in `language/dimensions.py` — both its dim *set* and its verdict on an operand that lacks the dim being reduced along, which lowering asks for rather than deciding again — and the dense-label assignment that gives a coordinate its solver index lives only in diff --git a/examples/walkthrough.py b/examples/walkthrough.py index 0c4bca24..a27af396 100644 --- a/examples/walkthrough.py +++ b/examples/walkthrough.py @@ -27,8 +27,8 @@ import polars as pl import lpspec as lps -from lpspec.expansion import parse_and_expand -from lpspec.expression_parser import parse_expression +from lpspec.language.expansion import parse_and_expand +from lpspec.language.expression_parser import parse_expression from lpspec.lowering import lower_program from lpspec.relational.executor import PolarsExecutor from lpspec.sources import tidy_sources diff --git a/pyproject.toml b/pyproject.toml index 92feae51..88e1bbfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,7 +165,7 @@ extend-fixable = ["B", "SIM", "RUF", "C4", "UP"] "bench/**/*.py" = ["T20"] # a harness reports by printing; stdout is its interface "tools/**/*.py" = ["T20"] # same: a generator says what it wrote "tests/golden/__main__.py" = ["T20"] # likewise a generator, and only reachable as a command -"src/lpspec/where_parser.py" = ["N806"] # grammar tokens (NOT/AND/OR) are named after keywords +"src/lpspec/language/where_parser.py" = ["N806"] # grammar tokens (NOT/AND/OR) are named after keywords [tool.ruff.format] quote-style = "single" diff --git a/src/lpspec/__init__.py b/src/lpspec/__init__.py index 29b4d54a..7a67ec52 100644 --- a/src/lpspec/__init__.py +++ b/src/lpspec/__init__.py @@ -27,7 +27,7 @@ PiecewiseExpansionError, SchemaError, ) -from lpspec.schema import MathSchema +from lpspec.language.schema import MathSchema from lpspec.typeset import SymbolTable, to_latex, to_markdown, to_typst __all__ = [ diff --git a/src/lpspec/api.py b/src/lpspec/api.py index cea35907..70e8e3eb 100644 --- a/src/lpspec/api.py +++ b/src/lpspec/api.py @@ -31,13 +31,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from lpspec._yaml import read_yaml +from lpspec.language._yaml import read_yaml +from lpspec.language.schema import MathSchema +from lpspec.language.validation import validate_expressions from lpspec.lowering import lower_program from lpspec.piecewise import expand_piecewise from lpspec.relational.executor import PolarsExecutor, Result -from lpspec.schema import MathSchema from lpspec.sources import tidy_sources -from lpspec.validation import validate_expressions if TYPE_CHECKING: from collections.abc import Mapping diff --git a/src/lpspec/language/__init__.py b/src/lpspec/language/__init__.py new file mode 100644 index 00000000..e6c068db --- /dev/null +++ b/src/lpspec/language/__init__.py @@ -0,0 +1,21 @@ +"""The language: what a YAML file may say, and what it means. + +Everything from the bytes on disk to a fully typed, dim-checked core AST — +the file reader, the schema, the two grammars, expansion, resolution, the dim +rules, and the load-time pass that runs them all. The AST this package +produces is the narrow waist of docs/ARCHITECTURE.md: everything downstream +reads it, and nothing downstream is visible from here. + +**The directory is the rule, in the direction the engine's is not.** Hard rule +2 says the engine never sees the schema or the AST; this is its mirror — +nothing under ``language/`` may import ``lowering``, ``piecewise``, +``sources``, ``api``, or any of the three consuming subpackages. What a model +*means* cannot depend on what any consumer does with it, which is what makes +``lps.check()`` a pass with no data and no plan, and a second consumer cheap. +``errors.py`` stays outside deliberately: it is the dependency-free leaf both +this package and the engine may import (``ENGINE_MAY_IMPORT``), and moving it +in would put the language's path on the engine's import list. + +``tests/test_architecture.py`` reads membership off the path, so a new +front-end module cannot land outside the fence by being spelled differently. +""" diff --git a/src/lpspec/_yaml.py b/src/lpspec/language/_yaml.py similarity index 100% rename from src/lpspec/_yaml.py rename to src/lpspec/language/_yaml.py diff --git a/src/lpspec/dimensions.py b/src/lpspec/language/dimensions.py similarity index 98% rename from src/lpspec/dimensions.py rename to src/lpspec/language/dimensions.py index 47213f99..a0f8d8f8 100644 --- a/src/lpspec/dimensions.py +++ b/src/lpspec/language/dimensions.py @@ -40,7 +40,7 @@ from typing import TYPE_CHECKING, assert_never from lpspec.errors import DimensionError -from lpspec.expression_parser import ( +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -55,8 +55,8 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.resolution import Namespace, expression_of, where_of -from lpspec.where_parser import ( +from lpspec.language.resolution import Namespace, expression_of, where_of +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, DimensionComparisonNode, @@ -73,7 +73,7 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema def dims_of( diff --git a/src/lpspec/expansion.py b/src/lpspec/language/expansion.py similarity index 98% rename from src/lpspec/expansion.py rename to src/lpspec/language/expansion.py index c1ca6293..76878ce1 100644 --- a/src/lpspec/expansion.py +++ b/src/lpspec/language/expansion.py @@ -40,7 +40,7 @@ from typing import TYPE_CHECKING, assert_never, overload from lpspec.errors import SchemaError -from lpspec.expression_parser import ( +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -60,7 +60,7 @@ if TYPE_CHECKING: from collections.abc import Callable - from lpspec.schema import MacroBlock, MathSchema + from lpspec.language.schema import MacroBlock, MathSchema #: Backstop against pathological nesting the cycle check cannot see. _MAX_DEPTH = 50 diff --git a/src/lpspec/expression_parser.py b/src/lpspec/language/expression_parser.py similarity index 100% rename from src/lpspec/expression_parser.py rename to src/lpspec/language/expression_parser.py diff --git a/src/lpspec/helpers.py b/src/lpspec/language/helpers.py similarity index 100% rename from src/lpspec/helpers.py rename to src/lpspec/language/helpers.py diff --git a/src/lpspec/resolution.py b/src/lpspec/language/resolution.py similarity index 98% rename from src/lpspec/resolution.py rename to src/lpspec/language/resolution.py index ae1e9992..292949d0 100644 --- a/src/lpspec/resolution.py +++ b/src/lpspec/language/resolution.py @@ -20,8 +20,8 @@ from typing import TYPE_CHECKING, assert_never from lpspec.errors import LanguageError -from lpspec.expansion import parse_and_expand -from lpspec.expression_parser import ( +from lpspec.language.expansion import parse_and_expand +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -36,8 +36,8 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.helpers import BUILTINS, EDGE_WRAP, call_shape_error, edge_error, unknown_helper_message -from lpspec.where_parser import ( +from lpspec.language.helpers import BUILTINS, EDGE_WRAP, call_shape_error, edge_error, unknown_helper_message +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, DimensionComparisonNode, @@ -55,7 +55,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Mapping - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema class Namespace: diff --git a/src/lpspec/schema.py b/src/lpspec/language/schema.py similarity index 99% rename from src/lpspec/schema.py rename to src/lpspec/language/schema.py index b4aa641d..8f6a1a79 100644 --- a/src/lpspec/schema.py +++ b/src/lpspec/language/schema.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from lpspec.helpers import BUILTIN_NAMES +from lpspec.language.helpers import BUILTIN_NAMES if TYPE_CHECKING: from collections.abc import Iterable diff --git a/src/lpspec/validation.py b/src/lpspec/language/validation.py similarity index 96% rename from src/lpspec/validation.py rename to src/lpspec/language/validation.py index 8576801d..8a707a47 100644 --- a/src/lpspec/validation.py +++ b/src/lpspec/language/validation.py @@ -23,10 +23,10 @@ from types import MappingProxyType from typing import TYPE_CHECKING, assert_never -from lpspec.dimensions import check_schema from lpspec.errors import SchemaError -from lpspec.expansion import expand, parse_and_expand, parse_template -from lpspec.expression_parser import ( +from lpspec.language.dimensions import check_schema +from lpspec.language.expansion import expand, parse_and_expand, parse_template +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -40,14 +40,14 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.helpers import BUILTINS, unknown_helper_message -from lpspec.resolution import Namespace, resolve_expression, resolve_where -from lpspec.where_parser import parse_where +from lpspec.language.helpers import BUILTINS, unknown_helper_message +from lpspec.language.resolution import Namespace, resolve_expression, resolve_where +from lpspec.language.where_parser import parse_where if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema def validate_expressions( diff --git a/src/lpspec/where_parser.py b/src/lpspec/language/where_parser.py similarity index 100% rename from src/lpspec/where_parser.py rename to src/lpspec/language/where_parser.py diff --git a/src/lpspec/linopy/__init__.py b/src/lpspec/linopy/__init__.py index b2cbbe86..dcf3e603 100644 --- a/src/lpspec/linopy/__init__.py +++ b/src/lpspec/linopy/__init__.py @@ -47,8 +47,10 @@ from lpspec._notes import note -from lpspec._yaml import read_yaml from lpspec.errors import LanguageError +from lpspec.language._yaml import read_yaml +from lpspec.language.schema import MathSchema +from lpspec.language.validation import validate_expressions from lpspec.linopy.builder import build_model from lpspec.linopy.loader import ( build_dim_coords, @@ -57,8 +59,6 @@ load_parameters, ) from lpspec.piecewise import expand_piecewise, validate_piecewise_data -from lpspec.schema import MathSchema -from lpspec.validation import validate_expressions # **This lane speaks v1, and the option is global, so importing sets it.** # diff --git a/src/lpspec/linopy/builder.py b/src/lpspec/linopy/builder.py index 62d5471a..56f65ba0 100644 --- a/src/lpspec/linopy/builder.py +++ b/src/lpspec/linopy/builder.py @@ -17,7 +17,7 @@ from lpspec._notes import note from lpspec.errors import DataError, LanguageError, null_bounds_message -from lpspec.expression_parser import ( +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -31,11 +31,9 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.helpers import EDGE_WRAP, unknown_helper_message -from lpspec.linopy import semantics -from lpspec.linopy.loader import check_divisors_cover -from lpspec.resolution import Namespace, expression_of, where_of -from lpspec.where_parser import ( +from lpspec.language.helpers import EDGE_WRAP, unknown_helper_message +from lpspec.language.resolution import Namespace, expression_of, where_of +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, DimensionComparisonNode, @@ -48,6 +46,8 @@ VariableDefinedNode, WhereNode, ) +from lpspec.linopy import semantics +from lpspec.linopy.loader import check_divisors_cover if TYPE_CHECKING: from collections.abc import Callable, Hashable, Mapping @@ -55,7 +55,7 @@ import linopy import pandas as pd - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema # Mapping from YAML comparison operators to linopy sign strings _SIGN_MAP = {'==': '=', '<=': '<=', '>=': '>='} diff --git a/src/lpspec/linopy/loader.py b/src/lpspec/linopy/loader.py index b63d439a..34ab7f7f 100644 --- a/src/lpspec/linopy/loader.py +++ b/src/lpspec/linopy/loader.py @@ -9,7 +9,7 @@ import xarray as xr from lpspec.errors import DataError, duplicate_coordinate_message, sparse_divisor_message -from lpspec.expression_parser import ( +from lpspec.language.expression_parser import ( BinaryOperatorNode, ComparisonNode, FunctionCallNode, @@ -20,7 +20,7 @@ ) if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema def build_master_coords( diff --git a/src/lpspec/lowering.py b/src/lpspec/lowering.py index 17ba5ddb..881d6625 100644 --- a/src/lpspec/lowering.py +++ b/src/lpspec/lowering.py @@ -29,9 +29,9 @@ from typing import TYPE_CHECKING, assert_never -from lpspec.dimensions import dims_of from lpspec.errors import LanguageError -from lpspec.expression_parser import ( +from lpspec.language.dimensions import dims_of +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -45,10 +45,9 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.helpers import BUILTIN_NAMES, call_shape_error, edge_error -from lpspec.relational import plan -from lpspec.resolution import Namespace, expression_of, where_of -from lpspec.where_parser import ( +from lpspec.language.helpers import BUILTIN_NAMES, call_shape_error, edge_error +from lpspec.language.resolution import Namespace, expression_of, where_of +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, DimensionComparisonNode, @@ -61,9 +60,10 @@ VariableDefinedNode, WhereNode, ) +from lpspec.relational import plan if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema _SENSES = {'==', '<=', '>='} diff --git a/src/lpspec/piecewise.py b/src/lpspec/piecewise.py index 7180bcde..8a2a8ea2 100644 --- a/src/lpspec/piecewise.py +++ b/src/lpspec/piecewise.py @@ -42,13 +42,13 @@ from typing import TYPE_CHECKING, Any -from lpspec.dimensions import dims_of from lpspec.errors import LanguageError, PiecewiseExpansionError -from lpspec.expression_parser import ComparisonNode, parse_expression +from lpspec.language.dimensions import dims_of +from lpspec.language.expression_parser import ComparisonNode, parse_expression +from lpspec.language.resolution import Namespace, resolve_expression +from lpspec.language.schema import MathSchema, PiecewiseBlock from lpspec.lowering import check_core_subset from lpspec.relational.frames import as_frame -from lpspec.resolution import Namespace, resolve_expression -from lpspec.schema import MathSchema, PiecewiseBlock if TYPE_CHECKING: from collections.abc import Mapping diff --git a/src/lpspec/sources.py b/src/lpspec/sources.py index d2be39e9..5e90c7e3 100644 --- a/src/lpspec/sources.py +++ b/src/lpspec/sources.py @@ -24,7 +24,7 @@ from lpspec.relational.frames import as_frame, labels_frame if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema def tidy_sources( diff --git a/src/lpspec/typeset/__init__.py b/src/lpspec/typeset/__init__.py index c07ff9cf..327d0cba 100644 --- a/src/lpspec/typeset/__init__.py +++ b/src/lpspec/typeset/__init__.py @@ -42,8 +42,8 @@ from typing import TYPE_CHECKING, Any from lpspec.api import load_schema +from lpspec.language.resolution import Namespace from lpspec.piecewise import expand_piecewise -from lpspec.resolution import Namespace from lpspec.typeset.latex import LatexFormat from lpspec.typeset.markdown import MarkdownFormat from lpspec.typeset.symbols import Symbols, SymbolTable @@ -54,7 +54,7 @@ from collections.abc import Mapping from pathlib import Path - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema from lpspec.typeset.format import Format __all__ = ['FORMATS', 'SymbolTable', 'to_latex', 'to_markdown', 'to_typst', 'typeset'] diff --git a/src/lpspec/typeset/symbols.py b/src/lpspec/typeset/symbols.py index c00cce5f..b2e51756 100644 --- a/src/lpspec/typeset/symbols.py +++ b/src/lpspec/typeset/symbols.py @@ -19,11 +19,11 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from lpspec._yaml import read_yaml from lpspec.errors import SchemaError +from lpspec.language._yaml import read_yaml if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema from lpspec.typeset.format import Format __all__ = ['SymbolTable', 'Symbols'] diff --git a/src/lpspec/typeset/walk.py b/src/lpspec/typeset/walk.py index 2f7fa486..c099ae3a 100644 --- a/src/lpspec/typeset/walk.py +++ b/src/lpspec/typeset/walk.py @@ -17,8 +17,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, assert_never -from lpspec.dimensions import dims_of -from lpspec.expression_parser import ( +from lpspec.language.dimensions import dims_of +from lpspec.language.expression_parser import ( ArithmeticNode, BinaryOperatorNode, ComparisonNode, @@ -32,9 +32,8 @@ UnaryOperatorNode, VariableNode, ) -from lpspec.resolution import expression_of, where_of -from lpspec.typeset.format import Entry, Line -from lpspec.where_parser import ( +from lpspec.language.resolution import expression_of, where_of +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, DimensionComparisonNode, @@ -47,10 +46,11 @@ VariableDefinedNode, WhereNode, ) +from lpspec.typeset.format import Entry, Line if TYPE_CHECKING: - from lpspec.resolution import Namespace - from lpspec.schema import MathSchema + from lpspec.language.resolution import Namespace + from lpspec.language.schema import MathSchema from lpspec.typeset.format import Format from lpspec.typeset.symbols import Symbols diff --git a/tests/conftest.py b/tests/conftest.py index 092a15da..1d37fd7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,7 +27,7 @@ import pytest import yaml as pyyaml -from lpspec.schema import MathSchema +from lpspec.language.schema import MathSchema EXAMPLES_DIR = Path(__file__).parent.parent / 'examples' @@ -126,14 +126,14 @@ def resolved(text, schema): this: a raw `parse_expression` result still holds NameNodes, and both backends now assert those never reach them (resolution.py). """ - from lpspec.resolution import Namespace, expression_of + from lpspec.language.resolution import Namespace, expression_of return expression_of(text, schema, Namespace.of(schema), 't') def resolved_where(text, schema): """Parse + resolve a where string.""" - from lpspec.resolution import Namespace, where_of + from lpspec.language.resolution import Namespace, where_of return where_of(text, Namespace.of(schema), 't') diff --git a/tests/differential.py b/tests/differential.py index c82bdd6b..0817806d 100644 --- a/tests/differential.py +++ b/tests/differential.py @@ -45,8 +45,8 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping + from lpspec.language.schema import MathSchema from lpspec.relational.executor import Result - from lpspec.schema import MathSchema #: Both lanes hand the same numbers to the same solver, so they must agree to #: solver precision, not to a fudge factor. One tolerance, one place. diff --git a/tests/test_api.py b/tests/test_api.py index a291a543..d8a6e8d2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -291,7 +291,7 @@ def test_no_helper_registry_anywhere(): makes the differential tests an oracle rather than a comparison of dialects (docs/ARCHITECTURE.md, "The expressive ceiling"). """ - import lpspec.helpers as helpers + import lpspec.language.helpers as helpers assert not hasattr(lps, 'register') assert not hasattr(helpers, 'register') diff --git a/tests/test_architecture.py b/tests/test_architecture.py index 3ef51d02..2214eff2 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -148,10 +148,56 @@ def test_engine_is_isolated(): assert not offenders, f'engine reaches outside its subpackage: {offenders}' +#: What ``language/`` may reach: itself, and the same dependency-free leaves the +#: engine may reach. Both fences point at ``errors.py`` for the same reason — +#: one exception hierarchy, owned by neither side. Widening this is a decision, +#: exactly as widening ``ENGINE_MAY_IMPORT`` is. +LANGUAGE_MAY_IMPORT = ENGINE_MAY_IMPORT + + +def test_language_never_reaches_a_consumer(): + """Hard rule 1, the other direction: the waist is closed from the front. + + Hard rule 2 keeps the engine from seeing the schema or the AST. This is its + mirror: what a model *means* may not depend on what any consumer does with + it, so nothing under ``language/`` imports ``lowering``, ``piecewise``, + ``sources``, ``api``, or the relational / linopy / typeset subpackages. + + That is what makes ``lps.check()`` a pass with no data and no plan, and a + second consumer cheap rather than a second opinion. Membership is read off + the path, so a new front-end module cannot land outside the fence by being + spelled differently. + """ + offenders = {} + for path in (PKG / 'language').rglob('*.py'): + if '__pycache__' in path.parts: + continue + bad = [] + for node in ast.walk(ast.parse(path.read_text())): # lazy imports included — the rule is total + names = ( + [a.name for a in node.names] + if isinstance(node, ast.Import) + else [node.module] + if isinstance(node, ast.ImportFrom) and node.module + else [] + ) + bad += [ + n + for n in names + if n.startswith('lpspec') and not n.startswith('lpspec.language') and n not in LANGUAGE_MAY_IMPORT + ] + if bad: + offenders[str(path.relative_to(PKG))] = sorted(set(bad)) + assert not offenders, ( + f'the language reaches forward to a consumer: {offenders} — a front-end module ' + f'may not depend on what is done with the AST it produces' + ) + + def test_expansion_has_no_mutable_module_state(): """Hard rule 5: YAML files are self-contained — nothing importable may accumulate state that changes what a file means.""" - tree = ast.parse((PKG / 'expansion.py').read_text()) + tree = ast.parse((PKG / 'language' / 'expansion.py').read_text()) mutable = [] for node in tree.body: if isinstance(node, (ast.Assign, ast.AnnAssign)): @@ -193,7 +239,7 @@ def test_both_lanes_implement_exactly_the_closed_helper_set(): Read statically: ``linopy/builder.py`` imports xarray at module level (it is linopy lane), and this check must still run on a bare install. """ - from lpspec.helpers import BUILTIN_NAMES + from lpspec.language.helpers import BUILTIN_NAMES tree = ast.parse((PKG / 'linopy' / 'builder.py').read_text()) table = next( @@ -244,7 +290,7 @@ def test_every_schema_model_is_strict(): """A schema model that inherits BaseModel directly silently drops unknown keys, which turns a typo into a different model. Strictness lives on ``_StrictBlock``, so the check is that nothing bypasses it.""" - tree = ast.parse((PKG / 'schema.py').read_text()) + tree = ast.parse((PKG / 'language' / 'schema.py').read_text()) loose = [ node.name for node in tree.body diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index 0f77c151..b3967e27 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -10,13 +10,13 @@ import pytest -from lpspec.dimensions import DimensionError, check_schema, dims_of -from lpspec.resolution import Namespace, expression_of +from lpspec.language.dimensions import DimensionError, check_schema, dims_of +from lpspec.language.resolution import Namespace, expression_of from tests.conftest import override, schema_of from tools import constructs if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema #: A *network* dispatch model: `conftest.DISPATCH_MODEL` plus buses, so #: `group_sum` and per-bus loads are in scope. The dim rules are mostly about diff --git a/tests/test_doc_examples.py b/tests/test_doc_examples.py index e2d2b512..2a082956 100644 --- a/tests/test_doc_examples.py +++ b/tests/test_doc_examples.py @@ -50,8 +50,8 @@ import yaml import lpspec as lps +from lpspec.language.schema import MathSchema from lpspec.relational.executor import PolarsExecutor, Result -from lpspec.schema import MathSchema try: from lpspec import linopy as linopy_lane diff --git a/tests/test_expansion.py b/tests/test_expansion.py index 657364ce..b589d5ee 100644 --- a/tests/test_expansion.py +++ b/tests/test_expansion.py @@ -9,10 +9,10 @@ import numpy as np import pytest -from lpspec.expansion import parse_and_expand -from lpspec.expression_parser import parse_expression -from lpspec.schema import MathSchema -from lpspec.validation import validate_expressions +from lpspec.language.expansion import parse_and_expand +from lpspec.language.expression_parser import parse_expression +from lpspec.language.schema import MathSchema +from lpspec.language.validation import validate_expressions from tests.differential import differential from tests.oracle import pd diff --git a/tests/test_linopy_lane.py b/tests/test_linopy_lane.py index 096bb8e2..cbff4ab8 100644 --- a/tests/test_linopy_lane.py +++ b/tests/test_linopy_lane.py @@ -19,7 +19,7 @@ import pytest from lpspec.errors import DataError, LanguageError -from lpspec.schema import MathSchema +from lpspec.language.schema import MathSchema from tests.oracle import builder, linopy, loader, lpspec_linopy, pd, xr @@ -282,7 +282,7 @@ def gens(): def _resolved(text, parameters=('p_max',), dimensions=('g',)): """Resolve then evaluate — the evaluator no longer takes strings.""" - from lpspec.resolution import Namespace, where_of + from lpspec.language.resolution import Namespace, where_of return where_of(text, Namespace((), parameters, dimensions), 'test') diff --git a/tests/test_lowering.py b/tests/test_lowering.py index 44908d44..72047f56 100644 --- a/tests/test_lowering.py +++ b/tests/test_lowering.py @@ -14,6 +14,8 @@ import pytest from lpspec.errors import DataError, DimensionError, LanguageError +from lpspec.language.resolution import Namespace +from lpspec.language.schema import MathSchema from lpspec.lowering import _lower_expr, _lower_where, lower_program from lpspec.relational.plan import ( DimensionComparison, @@ -23,8 +25,6 @@ Sum, Variable, ) -from lpspec.resolution import Namespace -from lpspec.schema import MathSchema from lpspec.sources import tidy_sources from tests.conftest import resolved, schema_of from tests.differential import differential diff --git a/tests/test_parser.py b/tests/test_parser.py index ba3f2a3c..44d8ee74 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -8,7 +8,7 @@ import pytest -from lpspec.expression_parser import ( +from lpspec.language.expression_parser import ( BinaryOperatorNode, ComparisonNode, FunctionCallNode, @@ -17,7 +17,7 @@ UnaryOperatorNode, parse_expression, ) -from lpspec.where_parser import ( +from lpspec.language.where_parser import ( AndNode, BooleanLiteralNode, NotNode, diff --git a/tests/test_relational.py b/tests/test_relational.py index 92170986..c6e48567 100644 --- a/tests/test_relational.py +++ b/tests/test_relational.py @@ -16,6 +16,7 @@ import lpspec as lps from lpspec.errors import DataError, LanguageError +from lpspec.language.schema import MathSchema from lpspec.lowering import lower_program from lpspec.relational import ( PolarsExecutor, @@ -35,7 +36,6 @@ Variable, VariableDeclaration, ) -from lpspec.schema import MathSchema from tests.conftest import solve_lp_file from tests.differential import RTOL, differential from tests.oracle import linopy, pd, transport_eager_objective, xr diff --git a/tests/test_resolution.py b/tests/test_resolution.py index f4903534..2347994c 100644 --- a/tests/test_resolution.py +++ b/tests/test_resolution.py @@ -11,13 +11,13 @@ import pytest from lpspec.errors import LanguageError +from lpspec.language.resolution import Namespace, expression_of, where_of +from lpspec.language.validation import validate_expressions from lpspec.lowering import lower_program -from lpspec.resolution import Namespace, expression_of, where_of -from lpspec.validation import validate_expressions from tests.conftest import DISPATCH_MODEL, schema_of if TYPE_CHECKING: - from lpspec.schema import MathSchema + from lpspec.language.schema import MathSchema def _schema(**overrides) -> MathSchema: @@ -30,7 +30,7 @@ def _schema(**overrides) -> MathSchema: def test_no_unresolved_name_survives_the_pass(): - from lpspec.expression_parser import NameNode + from lpspec.language.expression_parser import NameNode schema = _schema() ast = expression_of('sum(p * cost, over=generator) == load', schema, Namespace.of(schema), 't') @@ -49,7 +49,7 @@ def walk(node): def test_names_are_typed_by_kind(): - from lpspec.expression_parser import DimensionNode, ParameterNode, VariableNode + from lpspec.language.expression_parser import DimensionNode, ParameterNode, VariableNode schema = _schema() ast = expression_of('sum(p * cost, over=generator)', schema, Namespace.of(schema), 't') diff --git a/tests/test_resolution_parity.py b/tests/test_resolution_parity.py index bb5e43e8..2a0c52f3 100644 --- a/tests/test_resolution_parity.py +++ b/tests/test_resolution_parity.py @@ -121,8 +121,8 @@ def test_every_resolved_predicate_is_parity_tested(): """ from typing import get_args - from lpspec.resolution import Namespace, where_of - from lpspec.where_parser import UnresolvedComparisonNode, UnresolvedNameNode, WhereNode + from lpspec.language.resolution import Namespace, where_of + from lpspec.language.where_parser import UnresolvedComparisonNode, UnresolvedNameNode, WhereNode unresolved = {UnresolvedNameNode, UnresolvedComparisonNode} # rewritten by resolution, never evaluated expected = set(get_args(WhereNode)) - unresolved diff --git a/tests/test_schema.py b/tests/test_schema.py index a158e270..7e79d976 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -10,7 +10,7 @@ import pytest from pydantic import ValidationError -from lpspec.schema import MathSchema +from lpspec.language.schema import MathSchema def test_empty_schema(): diff --git a/tests/test_validation.py b/tests/test_validation.py index ab68d2ae..ea36cc63 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -6,8 +6,8 @@ import pytest -from lpspec.schema import MathSchema -from lpspec.validation import validate_expressions +from lpspec.language.schema import MathSchema +from lpspec.language.validation import validate_expressions from tests.oracle import linopy, lpspec_linopy, pd diff --git a/tests/test_yaml_loading.py b/tests/test_yaml_loading.py index 7549da76..051e3e45 100644 --- a/tests/test_yaml_loading.py +++ b/tests/test_yaml_loading.py @@ -5,8 +5,8 @@ import pytest import lpspec as lps -from lpspec._yaml import read_yaml -from lpspec.schema import MathSchema +from lpspec.language._yaml import read_yaml +from lpspec.language.schema import MathSchema MODEL = """dimensions: snapshot: {dtype: int, values: [0, 1]} diff --git a/tools/gallery_math.py b/tools/gallery_math.py index 1e30c4ce..3fdcc344 100644 --- a/tools/gallery_math.py +++ b/tools/gallery_math.py @@ -58,7 +58,7 @@ import sys from pathlib import Path -from lpspec._yaml import read_yaml +from lpspec.language._yaml import read_yaml from lpspec.typeset import to_latex, to_markdown from tools.constructs import models