From 7bad3d64c1cbde223dfc258510dea35de3bd8294 Mon Sep 17 00:00:00 2001 From: LunaStev Date: Fri, 14 Aug 2026 18:30:28 +0900 Subject: [PATCH] Make architecture tests cross-target contracts Separate compile-only target validation from native test execution by adding an explicit runner=compile metadata contract. This lets architecture fixtures validate generated artifacts on any CI host instead of being skipped when the host architecture differs from the compiler target. Add strict metadata parsing that rejects malformed, duplicate, unknown, empty, or incompatible fields. Validate build/runner coupling, host-independent compile contracts, artifact emit requirements, successful compile expectations, and RISC-V ABI architecture binding before invoking the compiler. Inspect generated ELF objects for class, machine, and RISC-V floating-point ABI flags, and inspect assembly using instruction-token matching that excludes comments, directives, and labels. Convert dormant RISC-V syscall fixtures to RV64 object and assembly contracts and make existing build-only fixtures explicit. Add permanent negative regression tests for invalid metadata, wrong ELF machine/class/ABI, and misleading assembly text. Run these tests together with Python syntax validation in every Rust and release CI architecture job. --- .github/workflows/release.yml | 4 +- .github/workflows/rust.yml | 24 ++- tests/cases/test101.wave | 2 +- tests/cases/test107.wave | 2 +- tests/cases/test108.wave | 2 +- tests/cases/test54.wave | 2 +- tests/cases/test94.wave | 2 +- tests/cases/test96.wave | 2 +- tests/cases/test99.wave | 2 +- tools/run_tests.py | 112 ++++------- tools/test_contracts.py | 367 ++++++++++++++++++++++++++++++++++ tools/test_test_contracts.py | 153 ++++++++++++++ 12 files changed, 587 insertions(+), 87 deletions(-) create mode 100644 tools/test_contracts.py create mode 100644 tools/test_test_contracts.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7245c5f..548e225a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Run Rust tests run: cargo test --locked --all-targets --verbose diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fdd0c3c3..777ea990 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,7 +59,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -119,7 +121,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -199,7 +203,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -325,7 +331,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -387,7 +395,9 @@ jobs: run: cargo clippy --locked --all-targets -- -D warnings - name: Validate Python tooling - run: python3 -m py_compile x.py tools/run_tests.py + run: | + python3 -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python3 -m unittest tools.test_test_contracts - name: Build release compiler run: cargo build --locked --release --verbose @@ -508,7 +518,9 @@ jobs: env: PYTHONUTF8: "1" PYTHONIOENCODING: "utf-8" - run: python -m py_compile x.py tools/run_tests.py + run: | + python -m py_compile x.py tools/run_tests.py tools/test_contracts.py tools/test_test_contracts.py + python -m unittest tools.test_test_contracts - name: Build release compiler shell: msys2 {0} diff --git a/tests/cases/test101.wave b/tests/cases/test101.wave index 4979fbd5..f94b4980 100644 --- a/tests/cases/test101.wave +++ b/tests/cases/test101.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=asm, asm-contains=ecall|a7 fun do_recv(sockfd: i64, buf: ptr>, src: ptr>, srclen: ptr) -> i64 { var n: i64; asm { diff --git a/tests/cases/test107.wave b/tests/cases/test107.wave index 9bb597a9..92394a70 100644 --- a/tests/cases/test107.wave +++ b/tests/cases/test107.wave @@ -1,4 +1,4 @@ -// wave-test: mode=build, emit=obj +// wave-test: mode=build, runner=compile, emit=obj export(c, "wave_add_i32") fun add_i32(a: i32, b: i32) -> i32 { return a + b; diff --git a/tests/cases/test108.wave b/tests/cases/test108.wave index 24c1a71d..4127421c 100644 --- a/tests/cases/test108.wave +++ b/tests/cases/test108.wave @@ -1,4 +1,4 @@ -// wave-test: mode=build, target=x86_64-pc-windows-gnu, emit=obj, freestanding=true +// wave-test: mode=build, runner=compile, target=x86_64-pc-windows-gnu, emit=obj, freestanding=true const KERNEL_ENTRY_POINT: u64 = 0x200000; const KERNEL_IMAGE_SIZE: u64 = 4; diff --git a/tests/cases/test54.wave b/tests/cases/test54.wave index f27367de..2c77a1f8 100644 --- a/tests/cases/test54.wave +++ b/tests/cases/test54.wave @@ -1,4 +1,4 @@ -// wave-test: host-arch=x86_64, mode=build, target=x86_64-unknown-none-elf, emit=obj, freestanding=true +// wave-test: mode=build, runner=compile, target=x86_64-unknown-none-elf, emit=obj, freestanding=true, object-arch=x86_64, object-bits=64 fun main() { asm { "mov ah, 0x0e" diff --git a/tests/cases/test94.wave b/tests/cases/test94.wave index 8f125259..00fcfdfd 100644 --- a/tests/cases/test94.wave +++ b/tests/cases/test94.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=obj, object-arch=riscv64, object-bits=64, riscv-float-abi=lp64d fun main() { var dummy_ptr: ptr = "dummy"; var ret_val: i64; diff --git a/tests/cases/test96.wave b/tests/cases/test96.wave index 45d99389..2459a319 100644 --- a/tests/cases/test96.wave +++ b/tests/cases/test96.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=asm, asm-contains=ecall|a7 fun factorial_simple(n: i32) -> i32 { var result: i32 = 1; var i: i32 = 1; diff --git a/tests/cases/test99.wave b/tests/cases/test99.wave index fc85d9fd..e5045a9c 100644 --- a/tests/cases/test99.wave +++ b/tests/cases/test99.wave @@ -1,4 +1,4 @@ -// wave-test: host-os=linux, host-arch=riscv64 +// wave-test: mode=build, runner=compile, target=riscv64-unknown-linux-gnu, emit=obj, object-arch=riscv64, object-bits=64, riscv-float-abi=lp64d fun syscall_mmap(addr: ptr, length: i64, prot: i64, flags: i64, fd: i64, offset: i64) -> ptr { let r: ptr; asm { diff --git a/tools/run_tests.py b/tools/run_tests.py index a277405c..ae0b8c80 100644 --- a/tools/run_tests.py +++ b/tools/run_tests.py @@ -24,6 +24,19 @@ import tempfile import errno +try: + from tools.test_contracts import ( + normalize_arch, + parse_test_metadata as parse_test_metadata_file, + validate_compiled_artifact, + ) +except ModuleNotFoundError: + from test_contracts import ( + normalize_arch, + parse_test_metadata as parse_test_metadata_file, + validate_compiled_artifact, + ) + ROOT = Path(__file__).resolve().parent.parent TEST_DIR = ROOT / "tests" / "cases" @@ -79,18 +92,7 @@ def resolve_wavec() -> Path: results = [] HOST_OS = platform.system().lower() -HOST_ARCH = platform.machine().lower() - - -def normalize_arch(arch: str) -> str: - aliases = { - "amd64": "x86_64", - "arm64": "aarch64", - } - return aliases.get(arch.lower(), arch.lower()) - - -HOST_ARCH = normalize_arch(HOST_ARCH) +HOST_ARCH = normalize_arch(platform.machine()) TEST_OUTPUT_DIR = Path(tempfile.mkdtemp(prefix="wave-test-output-")) @@ -128,64 +130,13 @@ def iter_test_entries(): def parse_test_metadata(rel_path: str): - path = ROOT / rel_path - meta = { - "host_os": None, - "host_arch": None, - "mode": "run", - "target": None, - "emit": "obj", - "freestanding": False, - "expected_exit": 0, - "udp_input": False, - } - - try: - for line in path.read_text().splitlines(): - stripped = line.strip() - if not stripped.startswith("//"): - if stripped: - break - continue - - marker = "// wave-test:" - if not stripped.startswith(marker): - continue - - body = stripped[len(marker):].strip() - for item in body.split(","): - item = item.strip() - if not item or "=" not in item: - continue - key, value = item.split("=", 1) - key = key.strip() - value = value.strip() - if key == "host-os": - meta["host_os"] = value.lower() - elif key == "host-arch": - meta["host_arch"] = normalize_arch(value) - elif key == "mode": - meta["mode"] = value.lower() - elif key == "target": - meta["target"] = value - elif key == "emit": - meta["emit"] = value.lower() - elif key == "freestanding": - meta["freestanding"] = value.lower() in {"1", "true", "yes"} - elif key == "expected-exit": - meta["expected_exit"] = int(value) - elif key == "udp-input": - meta["udp_input"] = value.lower() in {"1", "true", "yes"} - except OSError: - pass - - return meta + return parse_test_metadata_file(ROOT / rel_path, rel_path) def skip_reason_for_metadata(name: str, rel_path: str): meta = parse_test_metadata(rel_path) - host_os = meta["host_os"] - host_arch = meta["host_arch"] + host_os = meta.host_os + host_arch = meta.host_arch if host_os and host_os != HOST_OS: return f"{name} requires host OS {host_os}, current host is {HOST_OS}" @@ -198,7 +149,7 @@ def skip_reason_for_metadata(name: str, rel_path: str): def command_for_test(name: str, rel_path: str): meta = parse_test_metadata(rel_path) - mode = meta["mode"] + mode = meta.mode if mode == "run": return [str(WAVEC), "run", rel_path] @@ -214,13 +165,13 @@ def command_for_test(name: str, rel_path: str): str(WAVEC), "build", rel_path, - f"--emit={meta['emit']}", + f"--emit={meta.emit}", "--out-dir", str(output_dir), ] - if meta["target"]: - cmd.extend(["--target", meta["target"]]) - if meta["freestanding"]: + if meta.target: + cmd.extend(["--target", meta.target]) + if meta.freestanding: cmd.append("--freestanding") return cmd @@ -318,7 +269,8 @@ def run_and_classify(name, rel_path, cmd): print(f"{CYAN}→ SKIP ({skip_reason}){RESET}\n") return 2 - expected_exit = parse_test_metadata(rel_path)["expected_exit"] + metadata = parse_test_metadata(rel_path) + expected_exit = metadata.expected_exit stdin_data = None if name == "test22.wave": @@ -331,7 +283,7 @@ def run_and_classify(name, rel_path, cmd): return run_test56_server(cmd) try: - if parse_test_metadata(rel_path)["udp_input"]: + if metadata.udp_input: threading.Thread( target=send_udp_test_input, daemon=True @@ -362,6 +314,17 @@ def run_and_classify(name, rel_path, cmd): if expected_exit != 0: print(f"{MAGENTA}→ PASS (expected exit={expected_exit}){RESET}\n") return 3 + artifact_error = validate_compiled_artifact( + name, + ROOT / rel_path, + TEST_OUTPUT_DIR, + metadata, + ) + if artifact_error: + print(f"{RED}→ FAIL (artifact contract){RESET}") + print(artifact_error) + print() + return 0 print(f"{GREEN}→ PASS{RESET}\n") return 1 @@ -412,6 +375,9 @@ def run_and_classify(name, rel_path, cmd): except KeyboardInterrupt: print(f"\n{YELLOW}Interrupted by user.{RESET}") sys.exit(130) +except ValueError as error: + print(f"{RED}invalid wave-test metadata: {error}{RESET}", file=sys.stderr) + sys.exit(2) finally: shutil.rmtree(TEST_OUTPUT_DIR, ignore_errors=True) diff --git a/tools/test_contracts.py b/tools/test_contracts.py new file mode 100644 index 00000000..97b826b0 --- /dev/null +++ b/tools/test_contracts.py @@ -0,0 +1,367 @@ +# This file is part of the Wave language project. +# Copyright (c) 2024–2026 Wave Foundation +# Copyright (c) 2024–2026 LunaStev and contributors +# +# This Source Code Form is subject to the terms of the +# Mozilla Public License, v. 2.0. +# If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# AI TRAINING NOTICE: Prohibited without prior written permission. No use for machine learning or generative AI training, fine-tuning, distillation, embedding, or dataset creation. + +from dataclasses import dataclass +from pathlib import Path +import re + + +ARCH_ALIASES = { + "amd64": "x86_64", + "arm64": "aarch64", +} + +ELF_MACHINES = { + "x86_64": 62, + "aarch64": 183, + "riscv64": 243, +} + +RISCV_FLOAT_ABI_FLAGS = { + "lp64": 0x0, + "lp64f": 0x2, + "lp64d": 0x4, +} + +ARTIFACT_SUFFIXES = { + "obj": ".o", + "asm": ".s", + "ir": ".ll", + "bc": ".bc", +} + + +@dataclass(frozen=True) +class TestMetadata: + host_os: str | None = None + host_arch: str | None = None + mode: str = "run" + runner: str = "native" + target: str | None = None + emit: str = "obj" + freestanding: bool = False + expected_exit: int = 0 + udp_input: bool = False + object_arch: str | None = None + object_bits: int | None = None + riscv_float_abi: str | None = None + asm_contains: tuple[str, ...] = () + asm_not_contains: tuple[str, ...] = () + + +def normalize_arch(arch: str) -> str: + lowered = arch.lower() + return ARCH_ALIASES.get(lowered, lowered) + + +def _parse_bool(key: str, value: str, display_path: str) -> bool: + lowered = value.lower() + if lowered in {"1", "true", "yes"}: + return True + if lowered in {"0", "false", "no"}: + return False + raise ValueError( + f"metadata '{key}' in {display_path} expects true/false, found '{value}'" + ) + + +def _parse_int(key: str, value: str, display_path: str) -> int: + try: + return int(value) + except ValueError as error: + raise ValueError( + f"metadata '{key}' in {display_path} expects an integer, found '{value}'" + ) from error + + +def _parse_patterns(key: str, value: str, display_path: str) -> tuple[str, ...]: + patterns = tuple(pattern.strip() for pattern in value.split("|") if pattern.strip()) + if not patterns: + raise ValueError(f"metadata '{key}' in {display_path} must not be empty") + return patterns + + +def parse_test_metadata(path: Path, display_path: str | None = None) -> TestMetadata: + display = display_path or str(path) + values = {} + marker = "// wave-test:" + + try: + lines = path.read_text().splitlines() + except OSError as error: + raise ValueError(f"failed to read wave-test metadata from {display}: {error}") from error + + for line in lines: + stripped = line.strip() + if not stripped.startswith("//"): + if stripped: + break + continue + if not stripped.startswith(marker): + continue + + body = stripped[len(marker):].strip() + if not body: + raise ValueError(f"empty wave-test metadata in {display}") + + for raw_item in body.split(","): + item = raw_item.strip() + if not item or "=" not in item: + raise ValueError(f"malformed wave-test metadata '{item}' in {display}") + key, value = (part.strip() for part in item.split("=", 1)) + if not key or not value: + raise ValueError(f"malformed wave-test metadata '{item}' in {display}") + if key in values: + raise ValueError(f"duplicate wave-test metadata key '{key}' in {display}") + values[key] = value + + converters = { + "host-os": lambda value: value.lower(), + "host-arch": normalize_arch, + "mode": lambda value: value.lower(), + "runner": lambda value: value.lower(), + "target": str, + "emit": lambda value: value.lower(), + "freestanding": lambda value: _parse_bool("freestanding", value, display), + "expected-exit": lambda value: _parse_int("expected-exit", value, display), + "udp-input": lambda value: _parse_bool("udp-input", value, display), + "object-arch": normalize_arch, + "object-bits": lambda value: _parse_int("object-bits", value, display), + "riscv-float-abi": lambda value: value.lower(), + "asm-contains": lambda value: _parse_patterns("asm-contains", value, display), + "asm-not-contains": lambda value: _parse_patterns( + "asm-not-contains", value, display + ), + } + + converted = {} + field_names = { + "host-os": "host_os", + "host-arch": "host_arch", + "expected-exit": "expected_exit", + "udp-input": "udp_input", + "object-arch": "object_arch", + "object-bits": "object_bits", + "riscv-float-abi": "riscv_float_abi", + "asm-contains": "asm_contains", + "asm-not-contains": "asm_not_contains", + } + for key, value in values.items(): + converter = converters.get(key) + if converter is None: + raise ValueError(f"unsupported wave-test metadata key '{key}' in {display}") + converted[field_names.get(key, key)] = converter(value) + + metadata = TestMetadata(**converted) + compile_keys = { + "target", + "emit", + "freestanding", + "object-arch", + "object-bits", + "riscv-float-abi", + "asm-contains", + "asm-not-contains", + } + if compile_keys.intersection(values) and metadata.runner != "compile": + raise ValueError( + f"compile artifact metadata requires runner 'compile' in {display}" + ) + validate_test_metadata(metadata, display) + return metadata + + +def validate_test_metadata(metadata: TestMetadata, display_path: str) -> None: + if metadata.mode not in {"run", "check", "build"}: + raise ValueError( + f"unsupported wave-test mode '{metadata.mode}' in {display_path}" + ) + if metadata.runner not in {"native", "compile"}: + raise ValueError( + f"unsupported wave-test runner '{metadata.runner}' in {display_path}" + ) + if (metadata.mode == "build") != (metadata.runner == "compile"): + raise ValueError( + f"mode 'build' and runner 'compile' must be used together in {display_path}" + ) + + compile_only_fields = any( + ( + metadata.target, + metadata.freestanding, + metadata.object_arch, + metadata.object_bits, + metadata.riscv_float_abi, + metadata.asm_contains, + metadata.asm_not_contains, + ) + ) + if compile_only_fields and metadata.runner != "compile": + raise ValueError( + f"compile artifact metadata requires runner 'compile' in {display_path}" + ) + if metadata.runner == "compile" and (metadata.host_os or metadata.host_arch): + raise ValueError( + f"compile runner must not depend on the host platform in {display_path}" + ) + if metadata.runner == "compile" and metadata.emit not in ARTIFACT_SUFFIXES: + raise ValueError( + f"unsupported compile artifact emit '{metadata.emit}' in {display_path}" + ) + if metadata.udp_input and (metadata.runner != "native" or metadata.mode != "run"): + raise ValueError( + f"udp-input requires native run mode in {display_path}" + ) + + object_contract = any( + (metadata.object_arch, metadata.object_bits, metadata.riscv_float_abi) + ) + if object_contract and metadata.emit != "obj": + raise ValueError(f"ELF object metadata requires emit=obj in {display_path}") + if metadata.object_arch and metadata.object_arch not in ELF_MACHINES: + raise ValueError( + f"unsupported object architecture '{metadata.object_arch}' in {display_path}" + ) + if metadata.object_bits is not None and metadata.object_bits not in {32, 64}: + raise ValueError( + f"unsupported object bit width '{metadata.object_bits}' in {display_path}" + ) + if metadata.riscv_float_abi: + if metadata.riscv_float_abi not in RISCV_FLOAT_ABI_FLAGS: + raise ValueError( + f"unsupported RISC-V float ABI '{metadata.riscv_float_abi}' in {display_path}" + ) + if metadata.object_arch != "riscv64": + raise ValueError( + f"riscv-float-abi requires object-arch=riscv64 in {display_path}" + ) + + assembly_contract = metadata.asm_contains or metadata.asm_not_contains + if assembly_contract and metadata.emit != "asm": + raise ValueError(f"assembly pattern metadata requires emit=asm in {display_path}") + if metadata.expected_exit != 0 and (object_contract or assembly_contract): + raise ValueError( + f"artifact expectations require a successful compile in {display_path}" + ) + + +def artifact_path_for_test( + name: str, source_path: Path, output_root: Path, metadata: TestMetadata +) -> Path: + suffix = ARTIFACT_SUFFIXES[metadata.emit] + output_dir = output_root / name.replace(" ", "-") + return output_dir / f"{source_path.stem}{suffix}" + + +def read_elf_contract(path: Path): + try: + data = path.read_bytes() + except OSError as error: + return None, f"failed to read artifact {path}: {error}" + + if len(data) < 52 or data[:4] != b"\x7fELF": + return None, f"expected ELF object artifact, found {path}" + + elf_class = data[4] + encoding = data[5] + if encoding == 1: + byteorder = "little" + elif encoding == 2: + byteorder = "big" + else: + return None, f"invalid ELF data encoding {encoding} in {path}" + + flags_offset = {1: 36, 2: 48}.get(elf_class) + if flags_offset is None or len(data) < flags_offset + 4: + return None, f"invalid or truncated ELF header in {path}" + + return { + "bits": {1: 32, 2: 64}[elf_class], + "machine": int.from_bytes(data[18:20], byteorder), + "flags": int.from_bytes(data[flags_offset:flags_offset + 4], byteorder), + }, None + + +def _assembly_code_lines(assembly: str): + for raw_line in assembly.splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line or line.startswith(".") or line.endswith(":"): + continue + yield line + + +def _assembly_contains(assembly: str, pattern: str) -> bool: + token = re.compile( + rf"(?