From d194d687839b7d662466aeb550db44b652ed7d81 Mon Sep 17 00:00:00 2001 From: said Date: Thu, 4 Jun 2026 05:21:43 +0100 Subject: [PATCH] docs --- README.md | 37 +- docs/README.md | 32 +- docs/c_parser.md | 4 +- docs/{developer.md => developper_guide.md} | 172 +++- docs/examples.md | 622 +++++++++++ docs/fortran_parser.md | 6 +- docs/semantics.md | 69 +- docs/tutorial.md | 488 +++++++++ docs/user.md | 1080 -------------------- tests/tools/test_documentation_examples.py | 191 ++++ 10 files changed, 1547 insertions(+), 1154 deletions(-) rename docs/{developer.md => developper_guide.md} (81%) create mode 100644 docs/examples.md create mode 100644 docs/tutorial.md delete mode 100644 docs/user.md create mode 100644 tests/tools/test_documentation_examples.py diff --git a/README.md b/README.md index e3aa0e864..b8183cbca 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,16 @@ enough information for future wrapper generation. [![Tests](https://github.com/PyNumLab/x2py/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/PyNumLab/x2py/actions/workflows/tests.yml) [![codecov](https://codecov.io/gh/PyNumLab/x2py/graph/badge.svg?token=QZRRCS5YO6)](https://codecov.io/gh/PyNumLab/x2py) -## Documentation Levels +## Documentation -Use the README for orientation and common commands. Then use the audience -specific docs when you need detail: +Use the README for orientation and common commands. Then continue with: -- [User documentation](docs/user.md): more CLI cases, Python API workflows, - `.pyi` format, datatype mappings, readiness reports, and user policy - responsibilities. -- [Developer documentation](docs/developer.md): project internals, source - ownership, parser/semantic contracts, fixture maintenance, and focused test - commands. +- [Tutorial](docs/tutorial.md): supported end-to-end Fortran and C workflows, + semantic `.pyi` editing, readiness, and current limitations. +- [Verified examples cookbook](docs/examples.md): more CLI commands, compiler + preprocessing recipes, Python API examples, and readiness blocker examples. +- [Developer guide](docs/developper_guide.md): project internals, support + evidence rules, source ownership, fixture maintenance, and focused tests. - [C parser reference](docs/c_parser.md) and [Fortran parser reference](docs/fortran_parser.md): parser-specific coverage, behavior, diagnostics, and maintenance notes. @@ -98,12 +97,14 @@ end module m1 Command: + ```bash python -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse ``` Expected output: + ```text File: tests/data/fortran/general/basic_subroutine.f90 Modules: 1 @@ -114,10 +115,12 @@ File: tests/data/fortran/general/basic_subroutine.f90 Generate exact native `.pyi` stubs from the same Fortran file: + ```bash python -m x2py tests/data/fortran/general/basic_subroutine.f90 --pyi ``` + ```python File: tests/data/fortran/general/basic_subroutine.f90 def add1( @@ -128,10 +131,12 @@ def add1( Check readiness for the same Fortran file: + ```bash python -m x2py tests/data/fortran/general/basic_subroutine.f90 --wrap-readiness ``` + ```text File: tests/data/fortran/general/basic_subroutine.f90 Source: fortran @@ -163,10 +168,12 @@ void fill_identity3(double a[static 3][3]); Command: + ```bash python -m x2py tests/data/c/general/math_api.h --language c --parse ``` + ```text File: tests/data/c/general/math_api.h Language: c @@ -183,10 +190,12 @@ File: tests/data/c/general/math_api.h Generate C `.pyi` stubs from the same header: + ```bash python -m x2py tests/data/c/general/math_api.h --language c --pyi ``` + ```python File: tests/data/c/general/math_api.h def norm2( @@ -213,10 +222,12 @@ def fill_identity3( Check C readiness for the same header: + ```bash python -m x2py tests/data/c/general/math_api.h --language c --wrap-readiness ``` + ```text File: tests/data/c/general/math_api.h Source: c @@ -230,6 +241,7 @@ File: tests/data/c/general/math_api.h Generate C semantic IR: + ```bash python -m x2py tests/data/c/general/math_api.h --language c --semantics ``` @@ -601,7 +613,8 @@ compiler-preprocessed path or build an equivalent preprocessing configuration. mapping, and wrap-readiness checks. - `x2py/`: package entrypoints, preprocessing, and CLI integration. - `tests/`: parser, semantic, CLI, fixture, and property tests. -- `docs/`: user, developer, parser, semantic, quality, and design references. +- `docs/`: tutorial, examples, developer, parser, semantic, quality, and + design references. ## Running Tests @@ -612,5 +625,5 @@ PYTHONPATH=. pytest -q ``` Focused commands for parser changes, semantic changes, CLI changes, fixture -regeneration, linting, and coverage are in -[Developer Documentation](docs/developer.md#testing-map). +regeneration, linting, and coverage are in the +[Developer Guide](docs/developper_guide.md#testing-map). diff --git a/docs/README.md b/docs/README.md index 57433d830..47b2efbf0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,27 +1,43 @@ # Documentation -Use the audience-specific entrypoints first: +Start with: -- [User documentation](user.md): CLI/API usage, parser output, diagnostics, - generated `.pyi` files, semantic contracts, and readiness. -- [Developer documentation](developer.md): parser and semantic implementation - references, focused test files, fixture generators, CLI test commands, and - maintenance workflows. +- [Tutorial](tutorial.md): the supported end-to-end user workflow, semantic + `.pyi` editing, readiness, and current boundaries. +- [Verified examples cookbook](examples.md): copy-paste CLI commands, compiler + preprocessing recipes, Python API snippets, and blocker examples. +- [Developer guide](developper_guide.md): implementation ownership, support + evidence rules, focused tests, fixture generators, and change workflows. The repository [`README.md`](../README.md) remains the user-facing project overview. Contribution and pull-request requirements remain in [`CONTRIBUTING.md`](../CONTRIBUTING.md). -## Reference Files +## Current Contract References - [C parser reference](c_parser.md) - [Fortran parser reference](fortran_parser.md) - [Semantic IR and `.pyi` reference](semantics.md) -- [Wrapper design notes](wrapper_design_notes.md) - [Diagnostic code registry](diagnostic_codes.md) + +These files identify implemented, maintained contracts. Any design-only +material inside them must be labeled explicitly. The tutorial and examples +should link back to the implemented sections instead of inventing broader +support claims. + +## Maintainer References + +- [Developer guide](developper_guide.md) - [Quality assurance](quality.md) + +## Design Documents + +- [Wrapper design notes](wrapper_design_notes.md) - [Semantic multilanguage wrapper runtime architecture](architecture/semantic_multilanguage_wrapper_runtime_architecture.md) +Design documents describe deferred or long-term wrapper decisions. They are +not evidence that runtime wrapper generation is currently implemented. + README files under `tests/` intentionally remain next to the fixtures or expected outputs they describe. They are local test-maintenance instructions, not general project documentation. diff --git a/docs/c_parser.md b/docs/c_parser.md index cc43d5f2b..872e64e37 100644 --- a/docs/c_parser.md +++ b/docs/c_parser.md @@ -22,8 +22,8 @@ guard. ## Purpose -The C parser frontend will be a wrapper-oriented source extraction system for -x2py. It should extract enough stable semantic information from C sources and +The C parser frontend is a wrapper-oriented source extraction system for +x2py. It extracts stable semantic information from C sources and headers to help create or update the semantic interface layer. The implementation must be grammar-style: lex and slice source into C grammar diff --git a/docs/developer.md b/docs/developper_guide.md similarity index 81% rename from docs/developer.md rename to docs/developper_guide.md index c78c1ee66..c161ad1dd 100644 --- a/docs/developer.md +++ b/docs/developper_guide.md @@ -1,11 +1,131 @@ -# Developer Documentation +# Developer Guide -This page is for changing x2py. It names the relevant implementation -references, manual commands, focused test files, and fixture generators for each -part of the project. +This guide is for changing x2py. It maps user-visible behavior to its owning +implementation and tests, then gives focused change and verification +workflows. + +Use the [tutorial](tutorial.md) and [examples cookbook](examples.md) to inspect +the public workflows before changing them. Use the parser and semantic +references for the detailed maintained contracts. + +## Start Here + +Install the project and QA dependencies: + +```bash +python3 -m pip install -e ".[qa]" +``` + +Run the smallest relevant test while iterating, then run the full suite: + +```bash +PYTHONPATH=. python3 -m pytest -q tests/parser/test_cli.py +PYTHONPATH=. python3 -m pytest -q +``` + +Before changing a public behavior, trace it through these layers: + +```text +public command or Python API + -> owning parser or CLI entrypoint + -> parser model + -> semantic conversion, when applicable + -> .pyi printer/loader, when applicable + -> readiness, when applicable + -> focused tests and maintained reference docs +``` + +For example, a new CLI stage option normally requires: + +1. A focused contract test in `tests/parser/test_cli.py`. +2. Dispatch or output routing in `x2py/cli.py`. +3. Preprocessing tests if the option changes source loading. +4. A copy-paste command in [examples.md](examples.md). +5. A tutorial update only when the main user workflow changes. + +## Support Evidence Rule + +Documentation must describe implemented behavior, not intended behavior. +Treat a support claim as established only when it is traceable to current +implementation plus one of these forms of evidence: + +- a focused test that proves the contract; +- a maintained fixture test that proves generated output; +- a repository command that has been run against a checked fixture; +- an explicit parser or semantic reference inventory backed by tests. + +Use these documentation roles consistently: + +| Document | Role | +| --- | --- | +| [tutorial.md](tutorial.md) | Main supported user workflow and boundaries | +| [examples.md](examples.md) | Copy-paste commands and Python API recipes | +| [c_parser.md](c_parser.md) | Maintained C frontend support inventory | +| [fortran_parser.md](fortran_parser.md) | Maintained Fortran frontend support inventory | +| [semantics.md](semantics.md) | Accepted semantic IR and `.pyi` contract | +| [wrapper_design_notes.md](wrapper_design_notes.md) | Clearly deferred wrapper policy, not current runtime support | + +When adding a user example: + +1. Prefer a checked repository fixture or a short inline source string. +2. Run the command or snippet from the repository root. +3. Add or identify the focused test that owns the behavior. +4. State limitations next to the example when metadata is preserved but not + executed, such as `@native_call` projection metadata. +5. Do not describe future wrapper generation as implemented support. + +### Automatically Verify Markdown Examples + +`tests/tools/test_documentation_examples.py` executes explicitly marked +`bash` CLI examples and `python` API snippets from `README.md` and Markdown +files under `docs/`. Bash examples must be `python3 -m x2py` commands; the +test replaces `python3` with the active test interpreter and runs them without +a shell. It rejects shell operators, output-writing options, and options that +select custom executables or preprocessing command templates. Python snippets +run with the active test interpreter. + +Mark a command that only needs to exit successfully: + +````markdown + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --semantics +``` +```` + +Mark a command whose stdout must match the documentation exactly: + +````markdown + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse +``` + + +```text +File: tests/data/fortran/general/basic_subroutine.f90 +... +``` +```` + +Use exact checks for stable human-readable output. Use run checks for large +JSON or semantic payloads whose detailed contract is already covered by +focused tests. The same markers can precede a `python` fenced block. Do not +mark placeholder commands, snippets that modify the checkout, +environment-dependent compiler recipes, or intentionally failing diagnostic +examples. + +Run the documentation checks directly: + +```bash +PYTHONPATH=. python3 -m pytest -q tests/tools/test_documentation_examples.py +``` ## References +- [Tutorial](tutorial.md): supported end-to-end user workflow and current + boundaries. +- [Verified examples cookbook](examples.md): CLI and Python API recipes. - [C parser reference](c_parser.md): C frontend scope, preprocessing and project policy, parser architecture, CLI behavior, semantic handoff, fixtures, and tests. @@ -23,9 +143,10 @@ part of the project. ## User-Facing Contract Internals -The user documentation describes CLI stages, `.pyi` syntax, datatype names, and -readiness reports. The developer task is to keep those user-visible contracts -stable, tested, and traceable to implementation files. +The tutorial, examples cookbook, and semantic reference describe CLI stages, +`.pyi` syntax, datatype names, and readiness reports. The developer task is to +keep those user-visible contracts stable, tested, and traceable to +implementation files. ### Source Ownership Map @@ -43,6 +164,7 @@ stable, tested, and traceable to implementation files. | `.pyi` loading/editing | `semantics/pyi_parser.py` | `tests/pyi/test_pyi_to_ir.py`, `tests/pyi/test_pyi_fixture_suite.py` | | Readiness reports | `semantics/readiness.py` | `tests/semantics/test_semantic_wrap_readiness.py`, `tests/semantics/test_wrap_readiness_fixture_suite.py` | | Public API exports | `x2py/__init__.py` | `tests/parser/test_parser_public_entrypoints.py`, `tests/parser/c/test_c_public_api_skeleton.py` | +| Executable Markdown examples | `README.md`, `docs/*.md` | `tests/tools/test_documentation_examples.py` | ### `.pyi` Contract Internals @@ -67,7 +189,8 @@ When changing `.pyi` syntax: 1. Add or update parser tests in `tests/pyi/test_pyi_to_ir.py`. 2. Add or update printer tests in `tests/semantics/test_pyi_printer.py`. 3. Update fixture tests only if the public generated contract changes. -4. Update [user.md](user.md) if users need to write or read the new syntax. +4. Update [tutorial.md](tutorial.md) or [examples.md](examples.md) if users + need to write or read the new syntax. 5. Update [semantics.md](semantics.md) for the full reference. ### Datatype Mapping Internals @@ -86,8 +209,9 @@ When changing datatype mapping: 2. Add `.pyi` printer/loader coverage if the emitted syntax changes. 3. Update semantic fixtures only when serialized semantic IR intentionally changes. -4. Update [user.md](user.md) and [semantics.md](semantics.md) so the visible - mapping stays accurate. +4. Update [semantics.md](semantics.md), plus + [tutorial.md](tutorial.md) or [examples.md](examples.md) when the visible + user workflow or examples change. ### Readiness Internals @@ -104,8 +228,8 @@ When adding a readiness blocker: `tests/semantics/test_c_semantic_readiness.py`. 4. Update readiness fixtures only if user-visible messages intentionally change. -5. Update [user.md](user.md) when the blocker is something users can fix by - editing `.pyi`. +5. Update [tutorial.md](tutorial.md) or [examples.md](examples.md) when the + blocker is something users can fix by editing `.pyi`. ### Parser To Wrapper Boundary @@ -347,9 +471,9 @@ the C parser. `semantics/c2ir.py` and add coverage in `tests/semantics/test_c2ir.py`. 7. If the generated `.pyi` changes, update `tests/semantics/test_pyi_printer.py` or `tests/pyi/test_pyi_fixture_suite.py`. -8. Update [c_parser.md](c_parser.md), [user.md](user.md), or - [semantics.md](semantics.md) if users or maintainers need to know the new - behavior. +8. Update [c_parser.md](c_parser.md), [tutorial.md](tutorial.md), + [examples.md](examples.md), or [semantics.md](semantics.md) if users or + maintainers need to know the new behavior. Focused verification: @@ -387,8 +511,8 @@ metadata item. and `tests/semantics/test_fortran2ir.py`. 7. If generated `.pyi` changes, update `tests/semantics/test_pyi_printer.py` and the relevant fixture tests. -8. Update [fortran_parser.md](fortran_parser.md), [user.md](user.md), or - [semantics.md](semantics.md) as needed. +8. Update [fortran_parser.md](fortran_parser.md), [tutorial.md](tutorial.md), + [examples.md](examples.md), or [semantics.md](semantics.md) as needed. Focused verification: @@ -409,7 +533,9 @@ Example target: map a new Fortran kind, C typedef, or target-probed C type. there is a deliberate schema decision. 4. If the emitted `.pyi` annotation changes, update `tests/semantics/test_pyi_printer.py` and `tests/pyi/test_pyi_to_ir.py`. -5. Update the datatype tables in [user.md](user.md) and [semantics.md](semantics.md). +5. Update the datatype tables in [semantics.md](semantics.md), and update + [tutorial.md](tutorial.md) or [examples.md](examples.md) when a visible + example changes. Focused verification: @@ -429,7 +555,8 @@ Example target: add a new `Annotated[...]` metadata item or projection helper. 5. Update semantic models in `semantics/models.py` only if the IR needs a new field or constraint. 6. Update readiness behavior if the new syntax resolves a blocker. -7. Update [user.md](user.md) and [semantics.md](semantics.md). +7. Update [semantics.md](semantics.md), plus [tutorial.md](tutorial.md) or + [examples.md](examples.md) when users need the new syntax in a workflow. Focused verification: @@ -456,7 +583,8 @@ Example target: report a new unsupported C/Fortran semantic contract clearly. python tests/semantics/generate_wrap_readiness_fixtures.py ``` -6. Update [user.md](user.md) if users can fix the blocker by editing `.pyi`. +6. Update [tutorial.md](tutorial.md) or [examples.md](examples.md) if users + can fix the blocker by editing `.pyi`. Focused verification: @@ -475,8 +603,8 @@ diagnostic formatting. 3. Keep Fortran package-specific CLI behavior in `fortran_parser/cli.py`. 4. If compiler preprocessing behavior changes, update `x2py/preprocessing.py` and preprocessing tests. -5. Update [user.md](user.md) for user-facing commands and - [developer.md](developer.md) for maintainer command maps. +5. Update [tutorial.md](tutorial.md) or [examples.md](examples.md) for + user-facing commands and this guide for maintainer command maps. Focused verification: diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 000000000..61d9c5ba9 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,622 @@ +# Verified Examples Cookbook + +This cookbook collects supported x2py commands and Python API patterns. The +repository fixture commands and inline Python snippets are covered by the +current test suite or can be run directly from the repository root. + +Start with the [tutorial](tutorial.md) if this is your first x2py workflow. +Use the [semantic reference](semantics.md) for the full accepted `.pyi` +contract. + +## Fixture Inputs + +The most useful small, checked examples are: + +| Purpose | Repository fixture | +| --- | --- | +| Basic Fortran procedure | `tests/data/fortran/general/basic_subroutine.f90` | +| Rich Fortran module, types, arrays, and visibility | `tests/data/fortran/general/modern_pyi_example.f90` | +| Basic C functions, pointers, and arrays | `tests/data/c/general/math_api.h` | +| Generated Fortran semantic interface | `tests/pyi/fixtures/general/modern_pyi_example.pyi` | +| Generated C semantic interface | `tests/pyi/fixtures/c/general/math_api.pyi` | + +## CLI Stage Examples + +### Parse + +Compact Fortran report: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse +``` + +Compact C report: + +```bash +python3 -m x2py tests/data/c/general/math_api.h --language c --parse +``` + +Full parser payload: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse --json +python3 -m x2py tests/data/c/general/math_api.h --language c --parse --json +``` + +### Semantic IR + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --semantics +python3 -m x2py tests/data/c/general/math_api.h --language c --semantics +``` + +The semantic payload includes `semantic_modules` and generated `pyi` text. + +### `.pyi` Emission + +Print: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --pyi +python3 -m x2py tests/data/c/general/math_api.h --language c --pyi +``` + +Write an explicit interface file: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --pyi --out /tmp/basic_subroutine.pyi +``` + +Write one interface beside each input: + +```bash +python3 -m x2py path/to/fortran_sources --language fortran --pyi --out +``` + +### Readiness + +Human-readable readiness: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --wrap-readiness +python3 -m x2py tests/data/c/general/math_api.h --language c --wrap-readiness +python3 -m x2py tests/pyi/fixtures/general/basic_subroutine.pyi --wrap-readiness +``` + +Machine-readable readiness: + +```bash +python3 -m x2py tests/pyi/fixtures/general/basic_subroutine.pyi \ + --wrap-readiness --json +``` + +Read all `.pyi` files beneath a directory as one edited interface set. +Directories always require an explicit frontend, including `.pyi`-only +directories: + +```bash +python3 -m x2py path/to/interfaces --language fortran --wrap-readiness +``` + +### Combine Stages + +Print parser facts followed by readiness: + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --parse --wrap-readiness +``` + +Print a generated `.pyi` followed by readiness: + + +```bash +python3 -m x2py tests/data/c/general/math_api.h \ + --language c --pyi --wrap-readiness +``` + +Attach readiness to semantic output: + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --semantics --wrap-readiness +``` + +## Control Human-Readable Parse Output + +Fortran variable sections are compact by default. Expand them with +`--show-vars`: + + +```bash +python3 -m x2py tests/data/fortran/general/modern_pyi_example.f90 \ + --parse --show-vars +``` + +Limit every repeated section: + + +```bash +python3 -m x2py tests/data/fortran/general/modern_pyi_example.f90 \ + --parse --show-vars --print-limit 1 +``` + +This reports totals while showing one item from each repeated section: + + +```text +File: tests/data/fortran/general/modern_pyi_example.f90 + Modules: 1 + - module modern_math_physics (vars=2, uses=0) + Variables: 2 + - counter:integer[0] + ... 1 more variables + Derived types: 3 + - type particle (fields=3, methods=0) + Fields: 3 + - id:integer[0] + ... 2 more fields + ... 2 more derived types + Procedures: 7 + - subroutine init_particle(p:type(particle)[0], pid:integer[0], mass:real(8)[0], x:real(8)[0], y:real(8)[0], z:real(8)[0]) + ... 6 more procedures +``` + +`--show-vars` is Fortran-only. `--print-limit` works with human-readable C and +Fortran parse reports. + +## Input And Project Examples + +### Explicit Files + +```bash +python3 -m x2py src/types.f90 src/api.f90 --language fortran --parse +python3 -m x2py include/types.h include/api.h --language c --parse +``` + +### Directories + +Directories require an explicit frontend: + +```bash +python3 -m x2py src/fortran --language fortran --parse --print-limit 20 +python3 -m x2py src/c --language c --parse --print-limit 20 +``` + +Fortran directory discovery is recursive for recognized Fortran suffixes. C +directory discovery includes recognized C source, header, and preprocessed +input suffixes. + +### Unknown Suffixes + +Use an explicit frontend when the source suffix does not identify the +language: + +```bash +python3 -m x2py generated/api.source --language c --parse +python3 -m x2py generated/api.source --language fortran --parse +``` + +## Compiler Preprocessing + +The shared CLI preprocesses source before parsing. This is the supported path +for macros, conditional compilation, target flags, and native includes. + +### C Compiler And Flags + +```bash +python3 -m x2py include/api.h --language c --parse \ + --compiler clang \ + -I include \ + -D API_EXPORT= \ + -U LEGACY_API \ + --std c11 \ + --compiler-arg=--sysroot=/opt/sdk +``` + +### C Compilation Database + +```bash +python3 -m x2py src/api.c --language c --semantics \ + --compile-commands build/compile_commands.json +``` + +Extra wrapper-specific flags can be added to the selected database entry: + +```bash +python3 -m x2py src/api.c --language c --semantics \ + --compile-commands build/compile_commands.json \ + --compiler-arg=-DX2PY_SCAN=1 +``` + +### Fortran Compiler And Flags + +```bash +python3 -m x2py src/api.f90 --language fortran --pyi \ + --compiler gfortran \ + -I include \ + -D USE_MPI \ + --std f2008 \ + --compiler-arg=-fdefault-real-8 +``` + +### Custom Command Template + +Use a command template for an unsupported compiler family: + +```bash +python3 -m x2py include/api.h --language c --parse \ + --preprocessor-adapter command-template \ + --preprocess-template \ + 'cc -E {include_dirs} {defines} {undefs} {standard} {compiler_args} {source}' +``` + +Supported placeholders are `{source}`, `{include_dirs}`, `{defines}`, +`{undefs}`, `{standard}`, and `{compiler_args}`. + +### Include Exposure + +For C projects, reachable project includes are public by default and system +headers are private. Narrow or override that wrapper-facing surface: + +```bash +python3 -m x2py include/api.h --language c --pyi \ + --include-exposure roots-only \ + --public-include 'include/public/*' \ + --private-include 'vendor/*' +``` + +Private declarations remain available for internal type resolution. + +## Output File Examples + +Write one parser payload to an explicit path: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --parse --json --out /tmp/basic_subroutine.json +``` + +Write one parser payload beside each source: + +```bash +python3 -m x2py path/to/fortran_sources --language fortran --parse --out +``` + +Write an explicit `.pyi`: + +```bash +python3 -m x2py tests/data/c/general/math_api.h \ + --language c --pyi --out /tmp/math_api.pyi +``` + +`--out` requires a selected stage. `--json` and `--pyi` cannot both be used +with `--out`. + +## Diagnostic Examples + +Parser and preprocessing failures are rendered without a Python traceback by +default. Disable ANSI color for logs: + +```bash +python3 -m x2py path/to/source.f90 --parse --no-color +``` + +Re-raise an error with a traceback while debugging: + +```bash +python3 -m x2py path/to/source.f90 --parse --debug +``` + +Stable diagnostic categories are listed in +[diagnostic_codes.md](diagnostic_codes.md). + +## Python API Examples + +Direct parser APIs accept controlled source strings and paths. They do not run +the shared CLI compiler preprocessing pipeline. + +### Parse Inline Fortran + + +```python +from x2py import parse_fortran_file + +parsed = parse_fortran_file( + "subroutine ping(n)\n" + " integer, intent(in) :: n\n" + "end subroutine ping\n", + filename="inline.f90", +) + +print(parsed.procedures[0].name) # ping +``` + +Output: + + +```text +ping +``` + +### Parse Inline C + + +```python +from x2py import parse_c_file + +parsed = parse_c_file("int add(int a, int b);", filename="inline.h") + +print([function.name for function in parsed.functions]) # ['add'] +``` + +Output: + + +```text +['add'] +``` + +### Parse An In-Memory C Project + + +```python +from x2py import parse_c_project + +project = parse_c_project( + { + "types.h": "typedef int api_int;", + "api.h": '#include "types.h"\napi_int answer(void);', + } +) + +print(sorted(project.files)) # ['api.h', 'types.h'] +print(sorted(project.functions)) # ['answer'] +``` + +Output: + + +```text +['api.h', 'types.h'] +['answer'] +``` + +### Parse An In-Memory Fortran Project + + +```python +from x2py import parse_fortran_project + +project = parse_fortran_project( + { + "types.f90": "module types\nend module types\n", + "api.f90": ( + "module api\n" + " use types\n" + "end module api\n" + ), + } +) + +print(sorted(project.modules)) # ['api', 'types'] +``` + +Output: + + +```text +['api', 'types'] +``` + +### Convert C To Semantic IR, Emit `.pyi`, And Check Readiness + + +```python +from x2py import ( + assess_semantic_wrap_readiness, + c_file_to_semantic_modules, + emit_module_stubs, + parse_c_file, +) + +parsed = parse_c_file("int add(int a, int b);", filename="inline.h") +modules = c_file_to_semantic_modules(parsed) + +print(emit_module_stubs(modules)["inline"]) +print(assess_semantic_wrap_readiness(modules)["wrappable"]) +``` + +Output: + + +```text +def add( + a: Int32, + b: Int32 +) -> Int32: ... +True +``` + +### Check An Edited `.pyi` String + + +```python +from x2py import assess_semantic_wrap_readiness, parse_pyi_text + +module = parse_pyi_text( + """ +from typing import Callable + +def integrate( + objective: Callable[[Float64], Float64], + x0: Float64 +) -> Float64: ... +""", + module_name="solver", +) + +report = assess_semantic_wrap_readiness(module, source="solver.pyi") +print(report["wrappable"]) # True +``` + +Output: + + +```text +True +``` + +### Check `.pyi` Files Or Directories + +```python +from x2py import assess_pyi_wrap_readiness, load_pyi_modules + +modules = load_pyi_modules("path/to/interfaces") +report = assess_pyi_wrap_readiness("path/to/interfaces") + +print([module.name for module in modules]) +print(report["wrappable"]) +``` + +## Supported `.pyi` Examples + +These examples show semantic syntax accepted by the current loader. They are +contracts and metadata; they are not executable Python wrapper +implementations. + +### Scalars, References, And Arrays + +```python +def direct(value: Float64) -> Float64: ... +def inspect(value: Ptr(Const(Int32))) -> None: ... +def update(value: Ptr(Float64)) -> None: ... +def scale(n: Int32, values: Float64[n]) -> None: ... +def dot3(a: Const(Float64[3]), b: Const(Float64[3])) -> Float64: ... +``` + +### Constants, Visibility, And Classes + +```python +from typing import Final + +nmax: Final[Int32] = 32 +hidden_scale: private[Float64] + +class particle: + id: Int32 + mass: Float64 + position: Float64[3] + +@private +def helper(x: Ptr(Const(Int32))) -> None: ... +``` + +### Opaque Types + +```python +class context(Opaque): + pass + +def context_create() -> Ptr(context): ... +def context_destroy(ctx: Ptr(context)) -> None: ... +``` + +### Intent And Array Metadata + +```python +def fill( + values: Annotated[Float64[:], Intent("out")] +) -> None: ... + +def fill_matrix( + values: Annotated[Float64[3, 3], ORDER_F, Intent("out")] +) -> None: ... +``` + +### Complete Callback Signature + +```python +from typing import Callable + +def integrate( + objective: Callable[[Float64], Float64], + x0: Float64 +) -> Float64: ... +``` + +`Callable[..., Float64]` is accepted syntax but remains semantically +incomplete because the callback argument types are unknown. + +### Preserved Projection Metadata + +```python +@native_call([Arg(0), Arg(1), Return(0)]) +def add(a: Float64, b: Float64) -> Float64: ... +``` + +The current loader and printer preserve supported `@native_call` metadata. +x2py does not currently execute the projection or generate runtime wrapper +code. Use the [semantic reference](semantics.md) for the accepted projection +entries and limitations. + +## Readiness Blocker Examples + +### Missing Callback Signature + +```python +def integrate(objective: Procedure, x0: Float64) -> Float64: ... +``` + +This is blocked because callback argument order, argument types, and return +type are incomplete. Replacing `Procedure` with a complete supported +`Callable[[...], Return]` supplies the semantic signature. + +### Missing Compile-Time Constant + +```python +from typing import Final + +n: Final[Int32] + +def fill(values: Float64[n]) -> None: ... +``` + +This is blocked because `n` has no literal value. Supplying a value makes the +shape resolvable: + +```python +n: Final[Int32] = 16 +``` + +### Ambiguous C Pointer Policy + +```c +int read_values(double *values, size_t n); +``` + +The parser can preserve this signature, but semantic readiness reports +`c_pointer_ownership_ambiguous`: the declaration alone does not prove whether +the pointer is borrowed, owned, input, output, or in-place storage. Current +x2py does not invent that policy. + +### Unsupported C Variadic Function + +```c +int log_msg(const char *fmt, ...); +``` + +Semantic readiness reports `c_variadic_function`. Current x2py does not +generate a runtime wrapper for the variadic contract. + +## More References + +- [Tutorial](tutorial.md) +- [Semantic IR and `.pyi` reference](semantics.md) +- [Fortran parser reference](fortran_parser.md) +- [C parser reference](c_parser.md) +- [Diagnostic code registry](diagnostic_codes.md) +- [Developer guide](developper_guide.md) diff --git a/docs/fortran_parser.md b/docs/fortran_parser.md index 8b3de2210..13d337886 100644 --- a/docs/fortran_parser.md +++ b/docs/fortran_parser.md @@ -337,12 +337,14 @@ end module m1 Command: + ```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 +python -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse ``` Expected output: + ```text File: tests/data/fortran/general/basic_subroutine.f90 Modules: 1 @@ -355,10 +357,12 @@ The same command with `--show-vars` uses the variable-expanded report path. This fixture currently has no module variables to print, so the output remains compact: + ```bash python -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse --show-vars ``` + ```text File: tests/data/fortran/general/basic_subroutine.f90 Modules: 1 diff --git a/docs/semantics.md b/docs/semantics.md index 45e873c0d..5e59b6c9f 100644 --- a/docs/semantics.md +++ b/docs/semantics.md @@ -5,6 +5,10 @@ editable `.pyi` contracts, and the exact native C semantic stub rules. It is kept together because parser frontends should converge on one semantic model before wrapper generation makes language-specific lowering decisions. +Sections through [Deferred C Work](#deferred-c-work) describe current semantic +behavior. The final self-contained C runtime-contract section is explicitly a +design proposal and is not implemented wrapper support. + ## Datatype Mapping This document records the shared scalar datatype policy used when C and Fortran @@ -139,10 +143,10 @@ projection policy stays out of the generated `.pyi` until supplied by the semantic model or an edited interface. In particular, an unresolved typedef is not assumed to be opaque because its ABI representation is unknown. -## Wrapper `.pyi` Format +## Semantic `.pyi` Format The semantic `.pyi` format is a Python-valid view of x2py semantic IR. It is -language-neutral: Fortran and future C inputs use the same type, storage, +language-neutral: Fortran and C inputs use the same type, storage, pointer, array, layout and metadata notation. Source language differences are represented by contracts and metadata, not by separate syntax families. @@ -283,9 +287,9 @@ def legacy_matrix( ``` Assumed-shape arrays are stride-aware. A rank-one assumed-shape dummy is -emitted as a strided vector. Under the current generated-wrapper policy, a -rank-two or higher assumed-shape dummy retains Fortran orientation while -permitting strides: +emitted as a strided vector. Under the current generated semantic-interface +policy, a rank-two or higher assumed-shape dummy retains Fortran orientation +while permitting strides: ```python def vector(x: Float64[::Strided]) -> None: ... @@ -614,18 +618,25 @@ Future C conversion should use the same notation: by-value scalars as bare types, unrefined pointers as `Ptr(T)` or `Ptr(Const(T))`, and array notation only when a real array storage contract is known. -## Self-Contained C Semantic `.pyi` Contract +## Design Proposal: Self-Contained C Semantic `.pyi` Runtime Contract + +> **Status: design only, not implemented runtime support.** x2py currently +> parses C, converts the supported subset to semantic IR, emits and loads +> semantic `.pyi`, and reports readiness. It does not currently generate, +> lower, compile, or execute C wrappers. Every runtime behavior, wrapper error, +> and Phase 1/Phase 2 requirement below describes a proposed implementation +> target unless an earlier current-contract section explicitly says otherwise. -**Status:** Phase 1 implementation baseline -**Target:** Python wrappers for C libraries on a selected Linux ABI -**Primary requirement:** A semantic `.pyi` file plus a compiled library is -sufficient to generate a wrapper. C header parsing is optional input -generation, not a wrapper-generation dependency. +The proposed target is Python wrappers for C libraries on a selected Linux ABI. +Its primary design requirement is that a semantic `.pyi` file plus a compiled +library be sufficient to generate a wrapper, with C header parsing used only as +optional input generation. Related deferred policy is tracked in +[wrapper design notes](wrapper_design_notes.md). -### 1. Phase 1 Boundary +### 1. Proposed Phase 1 Boundary -Phase 1 implements the exact callable interface first. Python is intentionally -C-like at this stage: +The proposed Phase 1 would implement the exact callable interface first. +Python would intentionally remain C-like at this stage: - Every visible Python argument corresponds to one native C parameter, in the same order. @@ -638,12 +649,12 @@ C-like at this stage: - No argument is synthesized, reordered, omitted or converted into a Python result by the wrapper. -Therefore, Phase 1 does **not** implement or emit `@native_call`. +Therefore, the proposed Phase 1 would not implement or emit `@native_call`. The purpose of this ordering is to prove that x2py can describe, parse, lower and execute direct C signatures reliably before adding Pythonic adaptations. -### 2. Non-Negotiable Rules +### 2. Proposed Rules 1. The semantic `.pyi` must be sufficient to call every supported wrapped symbol without reading C source at build time. @@ -694,18 +705,18 @@ and execute direct C signatures reliably before adding Pythonic adaptations. 12. The current target is a selected Linux ABI. Cross-platform variation and non-default calling conventions are deferred. -### 3. Current Artifact +### 3. Proposed Artifact -The current compiler-facing artifact is: +The proposed compiler-facing artifact is: ```text module.x2py.pyi ``` -It may use x2py semantic types, but it contains only identity-callable +It may use x2py semantic types, but it would contain only identity-callable functions in Phase 1. -A clean `.pyi` for standard type checkers is not part of Phase 1. +A clean `.pyi` for standard type checkers is not part of the proposed Phase 1. ### 4. Scalar Types Passed By Value @@ -1197,12 +1208,12 @@ native representations are implemented explicitly: - variadic functions; - `void *` beyond an explicitly selected raw/byte-storage representation. -### 10. Phase 1 Unsupported Transformations +### 10. Transformations Excluded From Proposed Phase 1 Phase 1 must reject, or leave unresolved during optional C import generation, any interface that requires the wrapper to change the native function shape. -Unsupported now: +Excluded from the proposed Phase 1: | Desired behavior | Example C shape | Later mechanism | | --- | --- | --- | @@ -1218,10 +1229,10 @@ Unsupported now: The later syntax is retained as design direction only. It is not required by the Phase 1 parser, IR, printer or wrapper generator. -### 11. Required Phase 1 Readiness Errors +### 11. Proposed Phase 1 Runtime Errors -The wrapper generator or optional importer must report unsupported behavior -instead of silently changing the interface. +A future wrapper generator or optional importer would need to report +unsupported behavior instead of silently changing the interface. | Code | Condition | | --- | --- | @@ -1240,9 +1251,9 @@ instead of silently changing the interface. | `c_variadic_function_unsupported` | A variadic native function is requested. | | `c_calling_convention_unsupported` | A non-default calling convention is required. | -### 12. Phase 1 Parser And Wrapper Requirements +### 12. Proposed Phase 1 Parser And Wrapper Requirements -The Phase 1 implementation must: +The proposed Phase 1 implementation would need to: 1. Parse scalar annotations and direct `None`/scalar return annotations. 2. Parse unrefined one-level pointer forms `Ptr(T)` and `Ptr(Const(T))`, and @@ -1275,7 +1286,7 @@ The Phase 1 implementation must: later Pythonic mapping. 13. Never consult C source after a supported semantic `.pyi` has been parsed. -### 13. Phase 1 Tests +### 13. Proposed Phase 1 Runtime Tests #### 13.1 By-Value Scalar Identity @@ -1422,7 +1433,7 @@ later transformation syntax such as: def increment(value: Int) -> Returns["value", Int]: ... ``` -The supported Phase 1 spelling for the same C function is: +The proposed Phase 1 spelling for the same C function is: ```python def increment(value: Ptr(Int)) -> None: ... diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 000000000..084d2096d --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,488 @@ +# Tutorial + +This tutorial walks through the supported x2py pipeline from native source to +semantic readiness. The commands use version-controlled fixtures so they can be +run from the repository root. + +For more task-specific commands and Python snippets, use the +[examples cookbook](examples.md). For the complete semantic `.pyi` contract, +use the [semantic reference](semantics.md). + +## Current Scope + +x2py currently supports four user-facing stages: + +1. Parse wrapper-relevant Fortran or C declarations. +2. Convert parser facts to language-neutral semantic IR. +3. Emit an editable semantic `.pyi` interface. +4. Report whether that semantic interface has enough information for future + wrapper generation. + +x2py does **not** currently generate, compile, or load a runtime wrapper. +`Wrappable: yes` means the semantic contract has no known readiness blockers; +it does not mean a compiled Python extension already exists. + +The supported pipeline is: + +```text +Fortran or C source + -> parser facts + -> semantic IR + -> editable .pyi + -> semantic readiness report +``` + +Parsers preserve source facts. Semantic IR normalizes those facts. Edited +`.pyi` files are the user-controlled contract when source alone cannot express +enough policy. Readiness reports blockers rather than guessing ownership, +callback lifetime, ABI shims, or Python-visible projections. + +## Before You Start + +x2py requires Python 3.10 or newer. For native source input, the shared CLI +runs compiler preprocessing: + +- C defaults to `cc`. +- Fortran defaults to `gfortran`. +- Use `--compiler`, `--compile-commands`, or a custom preprocessing template + when the native project uses different flags or tools. + +Install the checkout and inspect the CLI: + +```bash +python3 -m pip install -e . +python3 -m x2py --help +``` + +The examples below use `python3`. Replace it with the Python 3.10+ executable +for your environment when necessary. After installation, the `x2py` console +command is equivalent to `python3 -m x2py`. + +## Fortran Walkthrough + +This walkthrough uses: + +```text +tests/data/fortran/general/basic_subroutine.f90 +``` + +### 1. Parse The Source + +Recognizable Fortran files do not require `--language fortran`: + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse +``` + +Expected output: + + +```text +File: tests/data/fortran/general/basic_subroutine.f90 + Modules: 1 + - module m1 (vars=0, uses=0) + Procedures: 1 + - subroutine add1(n:integer[0], x:real(8)[1]) +``` + +This is a compact source-fact report. It describes the native module and +procedure signature; it does not decide wrapper policy. + +### 2. Inspect Semantic IR + +Convert the parsed source to language-neutral semantic IR: + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --semantics +``` + +`--semantics` prints a machine-readable payload containing `semantic_modules` +and generated `pyi` text. Use it when another tool needs structured semantic +data. + +### 3. Generate The Editable Interface + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --pyi +``` + +Expected output: + + +```python +File: tests/data/fortran/general/basic_subroutine.f90 +def add1( + n: Ptr(Const(Int32)), + x: Float64[n] +) -> None: ... +``` + +The stub preserves the exact native contract: + +- `n` is a read-only scalar reference because the Fortran dummy argument is + not declared with `value`. +- `x` is a writable rank-one array whose extent is `n`. + +Write the stub to an explicit path: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --pyi --out basic_subroutine.pyi +``` + +Use `--pyi --out` without a path to write a `.pyi` beside each input source. + +### 4. Check Semantic Readiness + +Check readiness directly from source: + + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 --wrap-readiness +``` + +Expected output: + + +```text +File: tests/data/fortran/general/basic_subroutine.f90 + Source: fortran + Semantic modules: m1 + Wrappable: yes + Public functions: 1 + Public classes: 0 + Public variables: 0 + No semantic readiness blockers detected. +``` + +After editing a generated interface, check the `.pyi` instead: + +```bash +python3 -m x2py basic_subroutine.pyi --wrap-readiness +``` + +Readiness treats the edited `.pyi` contract as the source of truth. + +## C Walkthrough + +This walkthrough uses: + +```text +tests/data/c/general/math_api.h +``` + +C inputs require explicit C mode: + + +```bash +python3 -m x2py tests/data/c/general/math_api.h --language c --parse +``` + +Expected output: + + +```text +File: tests/data/c/general/math_api.h + Language: c + Functions: 4 + Structs: 0 + Unions: 0 + Enums: 0 + Typedefs: 0 + Variables: 0 + Macros: 0 + Includes: 0 + Diagnostics: 0 +``` + +Generate the semantic `.pyi`: + + +```bash +python3 -m x2py tests/data/c/general/math_api.h --language c --pyi +``` + +Expected output: + + +```python +File: tests/data/c/general/math_api.h +def norm2( + n: Int32, + x: Const(Float64[1]) +) -> Float64: ... + +def scale( + n: Int32, + alpha: Float64, + x: Float64[1] +) -> None: ... + +def dot( + n: Int32, + x: Ptr(Const(Float64)), + y: Ptr(Const(Float64)) +) -> Float64: ... + +def fill_identity3( + a: Float64[3, 3] +) -> None: ... +``` + +Check readiness: + + +```bash +python3 -m x2py tests/data/c/general/math_api.h \ + --language c --wrap-readiness +``` + +Expected output: + + +```text +File: tests/data/c/general/math_api.h + Source: c + Semantic modules: math_api + Wrappable: yes + Public functions: 4 + Public classes: 0 + Public variables: 0 + No semantic readiness blockers detected. +``` + +The C frontend supports wrapper-oriented declaration and signature extraction. +It is not a C++ frontend or a full compiler frontend. See the +[C parser reference](c_parser.md) for the maintained supported subset. + +## Choose A Stage + +| Goal | Command flag | Output | +| --- | --- | --- | +| Inspect native parser facts | `--parse` | Human-readable report | +| Consume full parser facts | `--parse --json` | Parser payload | +| Consume language-neutral facts | `--semantics` | Semantic payload | +| Create or inspect the editable contract | `--pyi` | Semantic `.pyi` text | +| Find missing semantic policy | `--wrap-readiness` | Readiness report | +| Script readiness decisions | `--wrap-readiness --json` | Readiness payload | + +Multiple stages can be useful together: + +```bash +python3 -m x2py tests/data/fortran/general/basic_subroutine.f90 \ + --parse --wrap-readiness + +python3 -m x2py tests/data/c/general/math_api.h \ + --language c --pyi --wrap-readiness +``` + +## Select Inputs And Language + +Language selection follows these supported rules: + +- Recognizable Fortran files can omit `--language`. +- `.pyi` readiness input can omit `--language`. +- C files require `--language c`. +- Directories and unknown-suffix source files require an explicit language. +- Explicit language selection must agree with recognizable source suffixes. + +Parse a directory recursively: + +```bash +python3 -m x2py path/to/fortran_sources --language fortran --parse +python3 -m x2py path/to/c_sources --language c --parse +``` + +Parse multiple explicit inputs: + +```bash +python3 -m x2py src/types.f90 src/api.f90 --language fortran --parse +python3 -m x2py include/types.h include/api.h --language c --parse +``` + +For the direct Python parser APIs, paths, source strings, project mappings, +path sequences, and directories are supported. Those direct APIs parse raw or +already-controlled input; they do not run the shared CLI compiler +preprocessing pipeline. + +## Use Native Project Compiler Flags + +The CLI preprocesses native source before parsing it. Pass the same important +flags used by the native build: + +```bash +python3 -m x2py include/api.h --language c --parse \ + --compiler clang \ + -I include \ + -D API_EXPORT= \ + --std c11 \ + --compiler-arg=--sysroot=/opt/sdk +``` + +Use a C compilation database when one is available: + +```bash +python3 -m x2py src/api.c --language c --semantics \ + --compile-commands build/compile_commands.json +``` + +For Fortran: + +```bash +python3 -m x2py src/api.f90 --language fortran --pyi \ + --compiler gfortran \ + -I include \ + -D USE_MPI \ + --std f2008 \ + --compiler-arg=-fdefault-real-8 +``` + +Compiler preprocessing preserves a recipe in machine-readable parser output, +including the selected compiler, arguments, includes, source mappings, and +diagnostics. See the [examples cookbook](examples.md#compiler-preprocessing) +for more supported preprocessing modes. + +## Edit A Semantic `.pyi` + +Generated `.pyi` files describe exact native contracts by default. They do not +silently hide native arguments, infer ownership, or turn output arguments into +Python return values. + +The loader accepts semantic interface syntax such as: + +```python +from typing import Callable, Final + +nmax: Final[Int32] = 32 + +class state: + count: Int32 + values: Float64[nmax] + +def integrate( + objective: Callable[[Float64], Float64], + x0: Float64 +) -> Float64: ... +``` + +A complete `Callable[[...], Return]` can resolve a callback-signature +readiness blocker. A placeholder such as `Procedure` or +`Callable[..., Return]` remains incomplete because argument types are unknown. + +Supported projection metadata such as `@native_call(...)` is parsed and +preserved, but x2py does not yet execute projections or generate runtime +wrapper code. See the [semantic reference](semantics.md) before writing custom +semantic annotations. + +## Use The Python API + +The package exports parser, semantic conversion, `.pyi`, and readiness helpers. + +Parse inline source: + + +```python +from x2py import parse_c_file, parse_fortran_file + +c_file = parse_c_file("int add(int a, int b);", filename="inline.h") +fortran_file = parse_fortran_file( + "subroutine ping()\nend subroutine ping\n", + filename="inline.f90", +) + +print([function.name for function in c_file.functions]) +print([procedure.name for procedure in fortran_file.procedures]) +``` + +Parse a project from an in-memory mapping: + + +```python +from x2py import parse_c_project + +project = parse_c_project( + { + "types.h": "typedef int api_int;", + "api.h": '#include "types.h"\napi_int answer(void);', + } +) + +print(sorted(project.files)) +print(sorted(project.functions)) +``` + +Convert, emit a stub, and check readiness: + + +```python +from x2py import ( + assess_semantic_wrap_readiness, + c_file_to_semantic_modules, + emit_module_stubs, + parse_c_file, +) + +parsed = parse_c_file("int add(int a, int b);", filename="inline.h") +modules = c_file_to_semantic_modules(parsed) +stubs = emit_module_stubs(modules) +report = assess_semantic_wrap_readiness(modules, source="inline.h") + +print(stubs["inline"]) +print(report["wrappable"]) +``` + +The [examples cookbook](examples.md#python-api-examples) contains additional +verified Python API workflows. + +## Understand Readiness + +Readiness is a semantic check. It can report blockers such as: + +- unresolved semantic types; +- unresolved array-shape symbols or missing compile-time constants; +- incomplete callback signatures; +- ambiguous C pointer ownership; +- C variadic or unspecified-parameter functions; +- unsupported union, bitfield, atomic, volatile, or ABI-sensitive contracts; +- an empty public API. + +When the missing information is expressible in supported semantic `.pyi` +syntax, edit the generated interface and rerun readiness. Some blockers require +future wrapper policy or implementation work and cannot currently be resolved +by an annotation. + +## Supported Boundaries + +Use x2py for the behavior implemented and tested today: + +- wrapper-relevant Fortran and C source-fact extraction; +- compiler-preprocessed CLI workflows; +- typed parser models and language-neutral semantic IR; +- semantic `.pyi` emission and loading; +- semantic readiness reporting. + +Do not assume current support for: + +- C++ parsing; +- full compiler-grade parsing or ABI validation; +- automatic pointer ownership or lifetime inference; +- automatic callback lifetime/threading policy; +- generated or compiled runtime wrappers; +- execution of `@native_call` projections. + +The maintained inventories are the +[Fortran parser reference](fortran_parser.md), +[C parser reference](c_parser.md), and +[semantic reference](semantics.md). + +## Continue Reading + +- [Verified examples cookbook](examples.md) +- [Semantic IR and `.pyi` reference](semantics.md) +- [Diagnostic code registry](diagnostic_codes.md) +- [Fortran parser reference](fortran_parser.md) +- [C parser reference](c_parser.md) +- [Developer guide](developper_guide.md) diff --git a/docs/user.md b/docs/user.md deleted file mode 100644 index 4898980df..000000000 --- a/docs/user.md +++ /dev/null @@ -1,1080 +0,0 @@ -# User Documentation - -This page is for using x2py from the command line, from Python, or by editing -generated `.pyi` files. It avoids maintainer-only fixture and implementation -workflow details. For implementation and testing workflow, use -[developer.md](developer.md). - -## What x2py Produces - -x2py has four user-facing stages: - -- Parse source and report parser facts. -- Convert parser facts to semantic IR. -- Generate editable `.pyi` semantic interface files. -- Check whether the semantic interface has enough information for wrapping. - -The parser is intentionally wrapper-focused. It records declarations, -signatures, types, array and pointer facts, source locations, and diagnostics. -It does not infer ownership, callback lifetime, ABI shims, or Pythonic -projections unless the user supplies that policy in `.pyi`. - -## Mental Model - -Think of x2py as a staged pipeline: - -```text -native source - -> parser facts - -> semantic IR - -> editable `.pyi` - -> readiness report - -> future wrapper generation -``` - -Each stage has a different job: - -- **Parser facts** are source-faithful. They preserve declarations, - signatures, source locations, includes/imports, diagnostics, and native type - facts. -- **Semantic IR** is language-neutral. C and Fortran declarations become the - same semantic concepts: modules, functions, classes, variables, scalar - types, arrays, pointers, constants, and blockers. -- **`.pyi`** is the editable user contract. It is where users can add missing - policy that source code alone cannot prove. -- **Readiness** checks whether the semantic contract is complete enough to - wrap. It reports blockers rather than guessing. -- **Wrapper generation** is the later stage that will consume a ready semantic - contract. - -The main rule is: generated `.pyi` files describe exact native contracts unless -you explicitly edit them. x2py should not silently hide native pointer/size -arguments, infer ownership, or invent callback lifetime policy. - -## Main CLI Workflows - -x2py uses the same stage flags for Fortran and C: - -- `--parse` prints parser facts. -- `--semantics` prints language-neutral semantic IR. -- `--pyi` prints editable `.pyi` interface text. -- `--wrap-readiness` checks whether the semantic contract is complete enough - for wrapping. - -Parse Fortran: - -```bash -python -m x2py path/to/file.f90 --parse -``` - -Parse C explicitly: - -```bash -python -m x2py path/to/header.h --language c --parse -``` - -Generate semantic IR: - -```bash -python -m x2py path/to/file.f90 --semantics -python -m x2py path/to/header.h --language c --semantics -``` - -Generate editable `.pyi` stubs: - -```bash -python -m x2py path/to/file.f90 --pyi -python -m x2py path/to/header.h --language c --pyi -``` - -Write output beside the source or to an explicit file: - -```bash -python -m x2py path/to/file.f90 --pyi --out -python -m x2py path/to/file.f90 --pyi --out interface.pyi -``` - -Check wrap readiness: - -```bash -python -m x2py path/to/file.f90 --wrap-readiness -python -m x2py path/to/interface.pyi --wrap-readiness -``` - -When a generated `.pyi` file needs policy that the parser cannot infer, edit -the stub and run readiness on the edited file. The edited `.pyi` is the -user-visible semantic contract. - -## Example Input Files - -The command examples below use these checked repository fixtures. Reading the -source, command, and output together should make each parser result -reproducible from the project root. - -`tests/data/fortran/general/basic_subroutine.f90` - -```fortran -module m1 -contains -subroutine add1(n, x) - integer, intent(in) :: n - real(kind=8), intent(inout), dimension(n) :: x -end subroutine add1 -end module m1 -``` - -`tests/data/c/general/math_api.h` - -```c -#ifndef X2PY_GENERAL_MATH_API_H -#define X2PY_GENERAL_MATH_API_H - -double norm2(int n, const double x[static 1]); -void scale(int n, double alpha, double x[static 1]); -double dot(int n, const double *restrict x, const double *restrict y); -void fill_identity3(double a[static 3][3]); - -#endif -``` - -## Compiler-Backed CLI Examples - -The CLI path uses compiler preprocessing for source parsing. C defaults to -`cc`, and Fortran defaults to `gfortran`, but wrapper work should pass the -same compiler and target flags used by the native project. - -### C With Include Paths, Macros, And Standard - -This command parses the `math_api.h` header shown in -[Example Input Files](#example-input-files): - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --parse --json \ - --compiler cc \ - -I tests/data/c/general \ - -D API_EXPORT= \ - --std c11 -``` - -The output is full parser JSON. The abbreviated excerpt below shows the -top-level fields that prove the file was compiler-preprocessed and which -recipe was used: - -```json -{ - "tests/data/c/general/math_api.h": { - "filename": "tests/data/c/general/math_api.h", - "language": "c", - "preprocessing": "compiler", - "preprocessing_recipe": { - "compiler": "cc", - "adapter": "gcc-compatible-c", - "argv": ["cc", "-E", "-x", "c", "-Itests/data/c/general", "-DAPI_EXPORT=", "-std=c11", "..."], - "include_dirs": ["tests/data/c/general"], - "defines": ["API_EXPORT="], - "standard": "c11", - "included_files": [{"path": "tests/data/c/general/math_api.h", "exposure": "public"}] - }, - "functions": [{"name": "norm2"}, {"name": "scale"}, {"name": "dot"}, {"name": "fill_identity3"}], - "diagnostics": [] - } -} -``` - -The same compiler flags flow through semantic, `.pyi`, and readiness stages: - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --pyi \ - --compiler cc \ - -I tests/data/c/general \ - -D API_EXPORT= \ - --std c11 -``` - -Expected `.pyi` output: - -```python -File: tests/data/c/general/math_api.h -def norm2( - n: Int32, - x: Const(Float64[1]) -) -> Float64: ... - -def scale( - n: Int32, - alpha: Float64, - x: Float64[1] -) -> None: ... - -def dot( - n: Int32, - x: Ptr(Const(Float64)), - y: Ptr(Const(Float64)) -) -> Float64: ... - -def fill_identity3( - a: Float64[3, 3] -) -> None: ... -``` - -Readiness for the same fixture: - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --wrap-readiness \ - --compiler cc \ - -I tests/data/c/general \ - -D API_EXPORT= \ - --std c11 -``` - -```text -File: tests/data/c/general/math_api.h - Source: c - Semantic modules: math_api - Wrappable: yes - Public functions: 4 - Public classes: 0 - Public variables: 0 - No semantic readiness blockers detected. -``` - -### C With Project Flags From `compile_commands.json` - -When a C project already has a compile database, use it so x2py sees the same -include paths, macros, target flags, and source language mode as the project: - -```bash -python -m x2py src/api.c --language c --parse \ - --compile-commands build/compile_commands.json -``` - -If the database entry needs extra wrapper-specific flags, add them explicitly: - -```bash -python -m x2py src/api.c --language c --semantics \ - --compile-commands build/compile_commands.json \ - --compiler-arg=--sysroot=/opt/sdk -``` - -### Fortran With Compiler Flags - -Use explicit Fortran compiler mode when source depends on CPP macros, -predefined compiler behavior, include paths, or target kind flags: - -This command parses the `basic_subroutine.f90` file shown in -[Example Input Files](#example-input-files): - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 \ - --language fortran \ - --parse --json \ - --compiler /usr/bin/gfortran \ - -I tests/data/fortran/general \ - -D USE_MPI \ - --std f2008 \ - --compiler-arg=-fdefault-real-8 -``` - -The output is full parser JSON. The abbreviated excerpt below shows the same -recipe shape: - -```json -{ - "tests/data/fortran/general/basic_subroutine.f90": { - "modules": [{"name": "m1"}], - "preprocessing_recipe": { - "language": "fortran", - "compiler": "/usr/bin/gfortran", - "adapter": "gnu-fortran", - "argv": ["/usr/bin/gfortran", "-E", "-cpp", "-Itests/data/fortran/general", "-DUSE_MPI", "-std=f2008", "-fdefault-real-8", "..."], - "include_dirs": ["tests/data/fortran/general"], - "defines": ["USE_MPI"], - "standard": "f2008", - "compiler_args": ["-fdefault-real-8"] - } - } -} -``` - -The same flags can generate `.pyi`: - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 \ - --language fortran \ - --pyi \ - --compiler /usr/bin/gfortran \ - -I tests/data/fortran/general \ - -D USE_MPI \ - --std f2008 \ - --compiler-arg=-fdefault-real-8 -``` - -```python -File: tests/data/fortran/general/basic_subroutine.f90 -def add1( - n: Ptr(Const(Int32)), - x: Float64[n] -) -> None: ... -``` - -### Include Exposure - -For C projects, included declarations can be public or private in the -wrapper-facing interface. By default, reachable project includes are public and -system headers are private. Use exposure flags when the public wrapper surface -should be narrower: - -```bash -python -m x2py include/api.h --language c --pyi \ - --include-exposure roots-only \ - --public-include 'include/public/*' \ - --private-include 'vendor/*' -``` - -Private declarations remain available internally for type resolution. Public -signatures that refer to private handle types can still use opaque dependency -stubs rather than exposing private data members. - -### Custom Preprocessor Adapter - -For unsupported compiler families, provide a command template. Placeholders -are expanded by x2py: - -```bash -python -m x2py path/to/api.h --language c --parse \ - --preprocessor-adapter command-template \ - --preprocess-template 'cc -E {include_dirs} {defines} {undefs} {standard} {compiler_args} {source}' -``` - -Supported placeholders include `{source}`, `{include_dirs}`, `{defines}`, -`{undefs}`, `{standard}`, and `{compiler_args}`. - -## End-To-End Tutorial - -This tutorial uses checked repository fixtures so the commands are easy to -reproduce from the project root. The Fortran steps use -`tests/data/fortran/general/basic_subroutine.f90`, shown in -[Example Input Files](#example-input-files). - -### 1. Parse A Fortran Source - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 --parse -``` - -Expected output: - -```text -File: tests/data/fortran/general/basic_subroutine.f90 - Modules: 1 - - module m1 (vars=0, uses=0) - Procedures: 1 - - subroutine add1(n:integer[0], x:real(8)[1]) -``` - -The parser is reporting native source facts only: modules, variables, procedure -signatures, argument types, ranks, and source diagnostics. - -### 2. Convert To Semantic IR - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 --semantics -``` - -Semantic output is the language-neutral view consumed by readiness and `.pyi` -generation. Use this when you need machine-readable contract data rather than -a human parse tree. - -### 3. Generate An Editable `.pyi` - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 --pyi -``` - -Generated stubs preserve the exact native interface. For example, scalar -Fortran dummy arguments without `value` appear as `Ptr(...)`, and arrays appear -with their storage contract: - -```python -File: tests/data/fortran/general/basic_subroutine.f90 -def add1( - n: Ptr(Const(Int32)), - x: Float64[n] -) -> None: ... -``` - -If the generated file is complete, you can use it as-is. If readiness reports -missing policy, edit the `.pyi` and make the contract explicit. - -### 4. Check Readiness - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 --wrap-readiness -``` - -For scripting, use JSON: - -```bash -python -m x2py tests/data/fortran/general/basic_subroutine.f90 --wrap-readiness --json -``` - -Look at `wrappable`, `wrappability_blockers`, `unit_blockers`, and -`why_not_wrappable`. - -### 5. Edit `.pyi` When Source Is Not Enough - -Some native APIs need user policy. For example, a C callback API may parse -successfully, but readiness may require a `Callable[...]` contract: - -```python -from typing import Callable - -def walk_items( - items: Ptr(Any), - visit: Callable[[Ptr(Any), Ptr(Any)], None], - userdata: Ptr(Any), -) -> None: ... -``` - -After editing, run readiness on the `.pyi` file: - -```bash -python -m x2py path/to/interface.pyi --wrap-readiness -``` - -## C End-To-End Tutorial - -Use explicit C mode for C headers and sources. These commands use -`tests/data/c/general/math_api.h`, shown in -[Example Input Files](#example-input-files): - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --parse -``` - -Expected parse output: - -```text -File: tests/data/c/general/math_api.h - Language: c - Functions: 4 - Structs: 0 - Unions: 0 - Enums: 0 - Typedefs: 0 - Variables: 0 - Macros: 0 - Includes: 0 - Diagnostics: 0 -``` - -Generate semantic IR JSON: - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --semantics -``` - -The semantic payload contains `semantic_modules` and a generated `pyi` string. -The module metadata includes `source_language: c` and `preprocessing: -compiler`. - -Generate `.pyi` directly: - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --pyi -``` - -The output preserves exact native contracts for the supported subset: - -```python -File: tests/data/c/general/math_api.h -def norm2( - n: Int32, - x: Const(Float64[1]) -) -> Float64: ... - -def scale( - n: Int32, - alpha: Float64, - x: Float64[1] -) -> None: ... - -def dot( - n: Int32, - x: Ptr(Const(Float64)), - y: Ptr(Const(Float64)) -) -> Float64: ... - -def fill_identity3( - a: Float64[3, 3] -) -> None: ... -``` - -Check readiness: - -```bash -python -m x2py tests/data/c/general/math_api.h --language c --wrap-readiness -``` - -```text -File: tests/data/c/general/math_api.h - Source: c - Semantic modules: math_api - Wrappable: yes - Public functions: 4 - Public classes: 0 - Public variables: 0 - No semantic readiness blockers detected. -``` - -## Python API Workflows - -Parse one Fortran file: - -```python -from x2py import parse_fortran_file - -parsed = parse_fortran_file("path/to/file.f90") -print([module.name for module in parsed.modules]) -``` - -Parse one C snippet or raw C file: - -```python -from x2py import parse_c_file - -parsed = parse_c_file("int add(int a, int b);", filename="api.h") -print([function.name for function in parsed.functions]) -``` - -Parse a C project explicitly: - -```python -from x2py import parse_c_project - -project = parse_c_project(["src/api.c", "include/api.h"], include_dirs=["include"]) -print(project.include_graph) -``` - -Generate semantic IR and assess readiness: - -```python -from x2py import ( - assess_semantic_wrap_readiness, - fortran_file_to_semantic_modules, - parse_fortran_file, -) - -parsed = parse_fortran_file("path/to/file.f90") -modules = fortran_file_to_semantic_modules(parsed, standalone_module_name="file") -report = assess_semantic_wrap_readiness(modules, source="path/to/file.f90") -print(report["wrappable"]) -``` - -## `.pyi` File Format - -x2py `.pyi` files are Python-valid semantic interface files. They describe the -native interface that x2py should wrap. Generated stubs are exact native -contracts by default: they do not hide pointer arguments, reorder arguments, or -turn output arguments into Python return values unless the `.pyi` explicitly -does that. - -The loader accepts ordinary imports used by the semantic file: - -```python -from typing import Annotated, Callable, Final -from other_module import particle -``` - -Imports matter when generated stubs refer to owner-module dependency stubs or -when an edited file supplies callback types. - -### Scalars - -Bare scalar types are direct values: - -```python -def dot_value(a: Float64, b: Float64) -> Float64: ... -``` - -Fortran scalar dummy arguments without `value` and C pointer-like scalar -storage are explicit pointer/reference contracts: - -```python -def inspect(value: Ptr(Const(Int32))) -> None: ... -def update(value: Ptr(Float64)) -> None: ... -``` - -`Ptr(Const(T))` means the native side receives a pointer/reference to read-only -storage. `Ptr(T)` means writable storage. - -### Function Returns - -A plain return type is a direct native return: - -```python -def norm2(n: Int32, x: Const(Float64[1])) -> Float64: ... -``` - -Functions that return multiple Python values can use tuple syntax: - -```python -def stats(x: Const(Float64[:])) -> tuple[Float64, Float64]: ... -``` - -For an exact native output argument, keep the native writable argument visible -unless a projection is explicitly supplied: - -```python -def get_count(out: Annotated[Ptr(Int32), Intent("out")]) -> None: ... -``` - -### Arrays - -Array storage uses NumPy-style subscriptions: - -```python -def scale(n: Ptr(Const(Int32)), x: Float64[n]) -> None: ... -def dot3(a: Const(Float64[3]), b: Const(Float64[3])) -> Float64: ... -``` - -Multidimensional Fortran-oriented storage uses `Annotated[..., ORDER_F]`: - -```python -def fill_identity3( - a: Annotated[Float64[3, 3], ORDER_F, Intent("out")] -) -> None: ... -``` - -Stride-aware arrays use slice syntax: - -```python -def scale_vector(v: Float64[::Strided], alpha: Ptr(Const(Float64))) -> None: ... -def update_matrix(a: Annotated[Float64[::Strided, ::Strided], ORDER_F]) -> None: ... -``` - -Rank-one arrays do not need an order marker. Multidimensional arrays default to -C order unless `ORDER_F` or `ORDER_ANY` is written. - -Shape expressions can reference earlier arguments or literal bounds: - -```python -def resize(n: Int32, values: Float64[1:n]) -> None: ... -def copy3(src: Const(Float64[3]), dst: Float64[3]) -> None: ... -``` - -For exact C pointer storage that does not carry a proven shape, use `Ptr(T)` or -`Ptr(Const(T))` instead of inventing a NumPy shape: - -```python -def dot(n: Int32, x: Ptr(Const(Float64)), y: Ptr(Const(Float64))) -> Float64: ... -``` - -### Constants And Globals - -Constants use `Final[...]`: - -```python -answer: Final[Int32] = 42 -pi: Final[Float64] -``` - -Module variables and global variables use normal annotations: - -```python -counter: Int32 -weights: Float64[16] -``` - -Visibility can be narrowed with `private[...]` or `@private`: - -```python -hidden_scale: private[Float64] - -@private -def helper(x: Ptr(Const(Int32))) -> None: ... -``` - -### Classes And Opaque Handles - -Derived types, structs, and opaque native handles are represented as classes: - -```python -class particle: - id: Int32 - mass: Float64 - position: Float64[3] -``` - -Opaque handles can be declared without exposing native layout: - -```python -class context(Opaque): - pass - -def context_destroy(ctx: Ptr(context)) -> None: ... -``` - -Class fields use the same scalar, array, pointer, and `Annotated[...]` syntax -as function arguments: - -```python -class vector3: - values: Float64[3] - -class particle: - id: Int32 - mass: Float64 - position: Float64[3] -``` - -### Intent, Allocatable, And Pointer Metadata - -Use `Annotated[...]` for non-dimensional metadata: - -```python -def fill(x: Annotated[Float64[:], Intent("out")]) -> None: ... -value: Annotated[Float64[:], ORDER_F, Allocatable] -view: Annotated[Float64[:], ORDER_F, Pointer] -``` - -`Intent("out")` is emitted when the exact native argument is an output -argument. For `intent(inout)`, the writable type itself normally carries the -contract. - -Generic constraints also use `Annotated[...]`: - -```python -value: Annotated[Int32, Bounded(1, 8), Finite] -alias: Annotated[Float64[:], Name("native_alias")] -``` - -`Name("...")` changes the native symbol name represented by the semantic -entry; the Python identifier can remain a valid Python spelling. - -### Callback Policy - -If the parser finds a callback shape but cannot prove its policy, edit the -stub with a `Callable[...]` signature: - -```python -from typing import Callable - -def walk_items( - items: Ptr(Any), - visit: Callable[[Ptr(Any), Ptr(Any)], None], - userdata: Ptr(Any), -) -> None: ... -``` - -The signature tells x2py the callable argument and return contract. Ownership, -lifetime, threading, and registration/unregistration policy still need to be -explicit when wrapper generation reaches those cases. - -Use the fully specified `Callable[[...], Return]` form when readiness reports a -callback blocker: - -```python -from typing import Callable - -def integrate(objective: Callable[[Float64], Float64], x0: Float64) -> Float64: ... -``` - -`Callable[..., Float64]` is accepted syntax, but it intentionally leaves -argument types unknown and may still be too weak for wrapper generation. - -### Pythonic Projections With `@native_call` - -The exact native form keeps every native argument visible: - -```python -def add( - a: Float64, - b: Float64, - c: Annotated[Ptr(Float64), Intent("out")] -) -> None: ... -``` - -A projected form can use `@native_call` to say how the Python signature maps to -native arguments: - -```python -@native_call([Arg(0), Arg(1), Return(0)]) -def add(a: Float64, b: Float64) -> Float64: ... -``` - -Here, native argument 0 comes from Python argument 0, native argument 1 comes -from Python argument 1, and native argument 2 is represented as Python return -value 0. Use projections only when you are intentionally changing the -Python-visible contract. Without explicit projection metadata, x2py keeps the -native contract. - -The supported projection entries today are: - -| Entry | Meaning | -| --- | --- | -| `Arg(i)` | Native argument comes from visible Python argument `i`. | -| `Return(i)` | Native argument is represented by Python return value `i`. | -| `Const(value)` | Hidden native argument is a literal value. | -| `Len(Arg(i))` | Hidden native argument is the length of argument `i`. | -| `Len(Return(i))` | Hidden native argument is the length of return value `i`. | -| `Arg(i).shape[d]` | Hidden native argument is dimension `d` of argument `i`. | -| `Work("name")` | Hidden native argument is named temporary workspace. | -| `Work("name").shape[d]` | Hidden native argument is a workspace dimension. | -| `IsPresent(Arg(i))` | Hidden native argument records whether optional argument `i` was supplied. | - -The list order is the native argument order. Positions are zero-based. - -Hidden literal and derived native arguments: - -```python -@native_call([ - Arg(0), - Const(1), - Len(Arg(0)), - Arg(0).shape[0], - IsPresent(Arg(1)), - Work("tmp"), -]) -def wrapper( - x: Float64[:], - b: Vector | None = None -) -> None: ... -``` - -Projection metadata can also reference return values or workspace dimensions: - -```python -@native_call([Len(Return(0)), Work("tmp").shape[1]]) -def produce_values() -> Float64[:]: ... -``` - -Method projections work the same way: - -```python -class particle: - @private - @native_call([Arg(0)]) - def reset(self: particle) -> Int32: ... -``` - -The `.pyi` loader and printer preserve this projection metadata. Wrapper -generation that actually hides pointer references, allocates workspace, -translates status returns, or applies coercions is still a later wrapper-stage -policy. Syntax such as `Ptr(Arg(...))`, `As[...]`, status checking, and -ownership helpers belongs to that future projection design; do not write it as -current accepted user syntax unless the implementation has added it. - -## Datatype Mapping - -These are the stable semantic names used in generated `.pyi` files and -readiness checks. - -| Semantic dtype | NumPy equivalent | Typical source spellings | -| --- | --- | --- | -| `Bool` | `numpy.bool_` | Fortran `logical`, C `_Bool` | -| `Int8` | `numpy.int8` | `integer(kind=1)`, `signed char`, `int8_t` | -| `Int16` | `numpy.int16` | `integer(kind=2)`, `short`, `int16_t` | -| `Int32` | `numpy.int32` | `integer`, `integer(c_int)`, `int`, `int32_t` | -| `Int64` | `numpy.int64` | `integer(kind=8)`, `long`, `long long`, `int64_t` | -| `UInt8` | `numpy.uint8` | `unsigned char`, `uint8_t` | -| `UInt16` | `numpy.uint16` | `unsigned short`, `uint16_t` | -| `UInt32` | `numpy.uint32` | `unsigned int`, `uint32_t` | -| `UInt64` | `numpy.uint64` | `unsigned long`, `unsigned long long`, `uint64_t` | -| `Float32` | `numpy.float32` | `real(kind=4)`, `real(c_float)`, `float` | -| `Float64` | `numpy.float64` | `real`, `real(kind=8)`, `real(c_double)`, `double` | -| `Float128` | `numpy.longdouble` | `real(kind=16)`, `long double` | -| `Complex64` | `numpy.complex64` | `complex(kind=4)`, `float _Complex` | -| `Complex128` | `numpy.complex128` | `complex`, `complex(kind=8)`, `double _Complex` | -| `Complex256` | `numpy.clongdouble` | `complex(kind=16)`, `long double _Complex` | -| `String` | `numpy.str_` or ABI byte storage | Fortran `character`, C `char *` policy | -| `SizeT` | `numpy.uintp` or probed unsigned width | C `size_t` | -| `Any` | `object` | `void *` pointees or intentionally opaque values | - -C standard-library typedefs such as `size_t`, `time_t`, and `FILE` can depend -on the target compiler and headers. Use the compiler/type probes when target -ABI precision matters. - -## Examples - -### Fortran Exact Stub - -Fortran source: - -```fortran -subroutine update(scale, value, result) - real(8), value, intent(in) :: scale - real(8), intent(inout) :: value - real(8), intent(out) :: result -end subroutine -``` - -Generated exact `.pyi` shape: - -```python -def update( - scale: Float64, - value: Ptr(Float64), - result: Annotated[Ptr(Float64), Intent("out")] -) -> None: ... -``` - -### Fortran Array Stub - -Fortran source: - -```fortran -subroutine fill_identity3(a) - real(8), intent(out) :: a(3, 3) -end subroutine -``` - -Generated `.pyi` shape: - -```python -def fill_identity3( - a: Annotated[Float64[3, 3], ORDER_F, Intent("out")] -) -> None: ... -``` - -### C Exact Stub - -C header: - -```c -int scale_values(size_t n, double values[n]); -double dot3(const double a[3], const double b[3]); -``` - -Generated `.pyi` shape for the supported exact subset: - -```python -def scale_values(n: SizeT, values: Float64[n]) -> Int32: ... -def dot3(a: Const(Float64[3]), b: Const(Float64[3])) -> Float64: ... -``` - -If x2py cannot prove the pointer/array relationship, readiness reports a -blocker instead of inventing a contract. - -### Opaque Handle Stub - -C header: - -```c -struct context; -struct context *context_create(void); -void context_destroy(struct context *ctx); -``` - -Generated `.pyi` shape: - -```python -class context(Opaque): - pass - -def context_create() -> Ptr(context): ... -def context_destroy(ctx: Ptr(context)) -> None: ... -``` - -### External Owner Stub - -When an imported Fortran derived type or external C opaque struct remains -outside the direct wrapping target, generated `.pyi` output can include an -owner-module dependency stub. - -Wrapping only `physics.f90` may produce: - -```python -# physics.pyi -from types_mod import particle - -def move(p: Ptr(particle)) -> None: ... -``` - -and a companion owner stub: - -```python -# types_mod.pyi -class particle(Opaque): - pass -``` - -The direct API keeps the correct import, and the owner stub gives readiness a -concrete semantic type. If the owner module is later part of the wrapping -target, replace the opaque placeholder with the edited concrete class. - -## Readiness Reports - -Readiness answers whether the semantic interface is complete enough to wrap. -Use JSON output when scripting: - -```bash -python -m x2py path/to/interface.pyi --wrap-readiness --json -``` - -Important fields include: - -- `wrappable`: final file-level answer. -- `n_modules`, `n_functions`, `n_classes`, `n_variables`: public API counts. -- `wrappability_blockers`: all blockers. -- `unit_blockers`: blockers grouped by function/class/module/global owner. -- `why_not_wrappable`: human-oriented blocker messages. - -Common blockers: - -- unresolved semantic types; -- missing compile-time values; -- unresolved shape symbols; -- ambiguous C pointer ownership; -- unsupported variadic functions; -- incomplete callback policy; -- unsupported unions or ABI-sensitive fields. - -When a blocker is policy-related, edit the `.pyi` to provide the missing -contract and rerun readiness. - -## User Responsibilities And Limits - -x2py deliberately avoids guessing policy that can change memory safety or API -meaning. Users should expect to provide policy for: - -- **Pointer ownership:** whether `T *` is borrowed, owned, nullable, mutable, - a scalar reference, or array storage. -- **Pointer/size relationships:** which `n`, `len`, `capacity`, or shape - argument describes which pointer. -- **Output buffers:** whether an output pointer remains a native argument or - becomes a Python return value through `@native_call`. -- **Callbacks:** callable signature, lifetime, context/userdata pairing, - registration/unregistration, threading, and exception policy. -- **ABI-sensitive structs/unions/bitfields:** whether direct passing is safe, - blocked, or needs a generated shim. -- **Compiler-dependent typedefs:** target-specific widths for types such as - `size_t`, `time_t`, and library handles. -- **Macro configurations:** parse each compiler-selected configuration after - preprocessing; do not expect raw macro expansion inside the parser. - -These limits are intentional. A readiness blocker is a request for a clearer -semantic contract, not just a parser failure. - -## Verified Example Sources - -These repository files and tests are useful examples to run or inspect: - -- Fortran user flow: - `tests/data/fortran/general/basic_subroutine.f90` -- Rich generated `.pyi` snapshot: - `tests/data/fortran/general/modern_pyi_example.f90` and - `tests/semantics/test_pyi_printer_modern_example.py` -- Edited `.pyi` syntax and `@native_call` examples: - `tests/pyi/test_pyi_to_ir.py` -- C parser and semantic examples: - `tests/data/c/general/math_api.h` and `tests/semantics/test_c2ir.py` -- Readiness examples: - `tests/semantics/test_semantic_wrap_readiness.py` and - `tests/semantics/test_c_semantic_readiness.py` - -## More Detail - -Use [semantics.md](semantics.md) for the full `.pyi` and semantic reference. -Use [diagnostic_codes.md](diagnostic_codes.md) for stable diagnostic code -names. Use [developer.md](developer.md) only when changing x2py itself. diff --git a/tests/tools/test_documentation_examples.py b/tests/tools/test_documentation_examples.py new file mode 100644 index 000000000..836639cc4 --- /dev/null +++ b/tests/tools/test_documentation_examples.py @@ -0,0 +1,191 @@ +"""Execute explicitly marked examples embedded in Markdown documentation.""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).parents[2] +DOC_PATHS = [ROOT / "README.md", *sorted((ROOT / "docs").rglob("*.md"))] +TEST_MARKER = re.compile(r"^\s*\s*$") +OUTPUT_MARKER = re.compile(r"^\s*\s*$") +FENCE_MARKER = re.compile(r"^\s*(`{3,}|~{3,})") +SHELL_OPERATORS = {"&&", "||", ";", "|", ">", ">>", "<", "2>", "2>>"} +DISALLOWED_OPTIONS = { + "--compile-commands", + "--compiler", + "--compiler-arg", + "--out", + "--preprocess-template", +} + + +@dataclass(frozen=True) +class DocumentationExample: + path: Path + line: int + mode: str + language: str + command: str + expected_output: str | None = None + + @property + def test_id(self) -> str: + return f"{self.path.relative_to(ROOT)}:{self.line}" + + +def _next_nonempty_line(lines: list[str], start: int) -> int: + index = start + while index < len(lines) and not lines[index].strip(): + index += 1 + return index + + +def _fenced_block(lines: list[str], start: int, *, language: str | None = None) -> tuple[str, int, str]: + start = _next_nonempty_line(lines, start) + if start >= len(lines) or not lines[start].startswith("```"): + raise AssertionError(f"expected a fenced block at line {start + 1}") + actual_language = lines[start][3:].strip() + if language is not None and actual_language != language: + raise AssertionError(f"expected a {language!r} fenced block at line {start + 1}, got {actual_language!r}") + + end = start + 1 + while end < len(lines) and lines[end].strip() != "```": + end += 1 + if end >= len(lines): + raise AssertionError(f"unclosed fenced block at line {start + 1}") + return "\n".join(lines[start + 1 : end]), end + 1, actual_language + + +def _logical_command(command_block: str, *, location: str) -> str: + command = re.sub(r"\\\n\s*", " ", command_block).strip() + if "\n" in command: + raise AssertionError(f"{location}: documentation tests must contain exactly one shell command") + return command + + +def _examples_from_path(path: Path) -> list[DocumentationExample]: + lines = path.read_text(encoding="utf-8").splitlines() + examples: list[DocumentationExample] = [] + index = 0 + + while index < len(lines): + marker = TEST_MARKER.match(lines[index]) + if marker is None: + if OUTPUT_MARKER.match(lines[index]): + raise AssertionError(f"{path.relative_to(ROOT)}:{index + 1}: output marker has no exact test") + fence = FENCE_MARKER.match(lines[index]) + if fence is not None: + token = fence.group(1) + index += 1 + while index < len(lines) and lines[index].strip() != token: + index += 1 + index += 1 + continue + + mode = marker.group(1) + marker_line = index + 1 + command_block, after_command, language = _fenced_block(lines, index + 1) + if language not in {"bash", "python"}: + raise AssertionError( + f"{path.relative_to(ROOT)}:{marker_line}: documentation tests require a bash or python fenced block" + ) + command = ( + _logical_command(command_block, location=f"{path.relative_to(ROOT)}:{marker_line}") + if language == "bash" + else command_block + ) + expected_output = None + index = after_command + + if mode == "exact": + while index < len(lines) and not OUTPUT_MARKER.match(lines[index]): + if TEST_MARKER.match(lines[index]): + raise AssertionError( + f"{path.relative_to(ROOT)}:{marker_line}: exact test is missing an output marker" + ) + index += 1 + if index >= len(lines): + raise AssertionError(f"{path.relative_to(ROOT)}:{marker_line}: exact test is missing an output marker") + expected_output, index, _output_language = _fenced_block(lines, index + 1) + + examples.append( + DocumentationExample( + path=path, + line=marker_line, + mode=mode, + language=language, + command=command, + expected_output=expected_output, + ) + ) + + return examples + + +DOCUMENTATION_EXAMPLES = [example for path in DOC_PATHS for example in _examples_from_path(path)] + + +def _command_argv(example: DocumentationExample) -> list[str]: + if example.language == "python": + return [sys.executable, "-c", example.command] + + argv = shlex.split(example.command) + if not argv or argv[0] not in {"python", "python3"} or argv[1:3] != ["-m", "x2py"]: + raise AssertionError(f"{example.test_id}: only 'python[3] -m x2py' commands are supported") + if any(argument in SHELL_OPERATORS for argument in argv): + raise AssertionError(f"{example.test_id}: shell operators are not supported") + if any( + argument == option or argument.startswith(f"{option}=") for argument in argv for option in DISALLOWED_OPTIONS + ): + raise AssertionError(f"{example.test_id}: output-writing and custom-executable options are not supported") + argv[0] = sys.executable + return argv + + +def test_documentation_has_automatically_verified_examples(): + assert DOCUMENTATION_EXAMPLES, "mark at least one Markdown example with x2py-doc-test" + assert any(example.mode == "exact" for example in DOCUMENTATION_EXAMPLES) + + +@pytest.mark.parametrize("path", DOC_PATHS, ids=lambda path: str(path.relative_to(ROOT))) +def test_documented_expected_output_labels_are_automatically_verified(path: Path): + lines = path.read_text(encoding="utf-8").splitlines() + for index, line in enumerate(lines): + if line.strip() not in {"Expected output:", "Output:"}: + continue + marker_index = _next_nonempty_line(lines, index + 1) + assert marker_index < len(lines) and OUTPUT_MARKER.match(lines[marker_index]), ( + f"{path.relative_to(ROOT)}:{index + 1}: documented output must use x2py-doc-test-output" + ) + + +@pytest.mark.parametrize("example", DOCUMENTATION_EXAMPLES, ids=lambda example: example.test_id) +def test_documentation_example(example: DocumentationExample): + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(filter(None, [str(ROOT), env.get("PYTHONPATH")])) + result = subprocess.run( + _command_argv(example), + cwd=ROOT, + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + assert result.returncode == 0, ( + f"{example.test_id}: command failed with status {result.returncode}\n" + f"command: {example.command}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + assert result.stderr == "", f"{example.test_id}: command wrote to stderr:\n{result.stderr}" + if example.mode == "exact": + assert result.stdout.rstrip("\n") == (example.expected_output or "").rstrip("\n")