diff --git a/.claude/rules/test-framework.md b/.claude/rules/test-framework.md index 041d372cf..400460a49 100644 --- a/.claude/rules/test-framework.md +++ b/.claude/rules/test-framework.md @@ -10,17 +10,15 @@ paths: 1. **Unit tests** (`tests/lean_spec/`) - Standard pytest tests for implementation 2. **Spec tests** (`tests/consensus/`) - Generate JSON test vectors via fillers - - *Note: `tests/execution/` infrastructure is ready for future execution layer work* **Test Filling Framework:** -- Layer-agnostic pytest plugin in `packages/testing/src/framework/pytest_plugins/filler.py` -- Layer-specific packages: `consensus_testing` (active) and `execution_testing` (future) +- Pytest plugin in `packages/testing/src/framework/pytest_plugins/filler.py` +- Consensus fixture package: `consensus_testing` - Write consensus spec tests using `state_transition_test` or `fork_choice_test` fixtures - These fixtures are type aliases that create test vectors when called - Run `uv run fill --fork=Lstar --clean -n auto` to generate consensus fixtures -- Use `--layer=execution` flag when execution layer is implemented -- Output goes to `fixtures/{layer}/{format}/{test_path}/...` +- Output goes to `fixtures/consensus/{format}/{test_path}/...` **Example spec test:** @@ -40,13 +38,12 @@ def test_block(state_transition_test: StateTransitionTestFiller) -> None: 3. `make_fixture()` executes the spec code (state transitions, fork choice steps) 4. Validates output against expectations (`StateExpectation`, `StoreChecks`) 5. Serializes to JSON via Pydantic's `model_dump(mode="json")` -6. Writes fixtures at session end to `fixtures/{layer}/{format}/{test_path}/...` +6. Writes fixtures at session end to `fixtures/consensus/{format}/{test_path}/...` -**Layer-specific architecture:** +**Package architecture:** -- `framework/` - Shared infrastructure (base classes, pytest plugin, CLI) -- `consensus_testing/` - Consensus layer fixtures, forks, builders -- `execution_testing/` - Execution layer fixtures, forks, builders +- `framework/` - Pytest plugin, CLI entry points, fork registry infrastructure +- `consensus_testing/` - Consensus fixtures, forks, builders - Regular pytest runs (`uv run pytest`) ignore spec tests - they only run via `fill` command **Serialization requirements:** @@ -54,10 +51,9 @@ def test_block(state_transition_test: StateTransitionTestFiller) -> None: - All spec types (State, Block, Uint64, etc.) must be Pydantic models - Custom types need `@field_serializer` or `model_serializer` for JSON output - SSZ types typically serialize to hex strings (e.g., `"0x1234..."`) -- Fixture models inherit from layer-specific base classes: - - Consensus: `BaseConsensusFixture` (in `consensus_testing/test_fixtures/base.py`) - - Execution: `BaseExecutionFixture` (in `execution_testing/test_fixtures/base.py`) - - Both use `CamelModel` for camelCase JSON output +- Fixture models inherit from `BaseConsensusFixture` (in + `consensus_testing/test_fixtures/base.py`), which uses `CamelModel` for + camelCase JSON output - Test the serialization: `fixture.model_dump(mode="json")` must produce valid JSON **Key fixture types:** @@ -65,4 +61,3 @@ def test_block(state_transition_test: StateTransitionTestFiller) -> None: - `StateTransitionTest` - Tests state transitions with blocks - `ForkChoiceTest` - Tests fork choice with steps (tick/block/attestation) - Selective validation via `StateExpectation` and `StoreChecks` (only validates fields you specify) - diff --git a/.claude/rules/workflow.md b/.claude/rules/workflow.md index 0d00fbcaa..570727f3b 100644 --- a/.claude/rules/workflow.md +++ b/.claude/rules/workflow.md @@ -7,8 +7,6 @@ uv sync # Install dependencies uv run pytest # Run unit tests uv run fill --fork=lstar --clean -n auto # Generate test vectors uv run fill --fork=lstar --clean -n auto --scheme=prod # Generate test vectors with production scheme -# Note: execution layer support is planned for future, infrastructure is ready -# for now, `--layer=consensus` is default and the only value used. ``` ## Code Quality @@ -27,5 +25,4 @@ just # List all available recipes - **Subspecs**: `src/lean_spec/subspecs/{subspec}/` - **Unit tests**: `tests/lean_spec/` (mirrors source structure) - **Consensus spec tests**: `tests/consensus/` (generates test vectors) -- **Execution spec tests**: `tests/execution/` (future - infrastructure ready) diff --git a/.claude/skills/spec-diff/SKILL.md b/.claude/skills/spec-diff/SKILL.md index d4f9179a1..23bafe2e5 100644 --- a/.claude/skills/spec-diff/SKILL.md +++ b/.claude/skills/spec-diff/SKILL.md @@ -11,8 +11,8 @@ Show what changed in the **spec code** (`src/lean_spec/`) and **consensus test v **Scope**: Protocol-level spec types, functions, containers, forkchoice logic, and the test fixtures that generate cross-client test vectors. -**Excluded**: Test framework infrastructure (`packages/testing/`, `consensus_testing/`, -`execution_testing/`), unit tests (`tests/lean_spec/`), interop tests (`tests/interop/`), +**Excluded**: Test framework infrastructure (`packages/testing/`, `consensus_testing/`), +unit tests (`tests/lean_spec/`), interop tests (`tests/interop/`), documentation (`docs/`), CI/tooling configs, and the node implementation layer (networking, sync, storage, node runner). diff --git a/packages/testing/src/consensus_testing/test_fixtures/base.py b/packages/testing/src/consensus_testing/test_fixtures/base.py index d3d8a1633..b8da5a9e6 100644 --- a/packages/testing/src/consensus_testing/test_fixtures/base.py +++ b/packages/testing/src/consensus_testing/test_fixtures/base.py @@ -1,22 +1,43 @@ """Base fixture definitions for consensus test formats.""" +import hashlib +import json +from functools import cached_property from typing import Any, ClassVar -from framework.test_fixtures import BaseFixture -from pydantic import field_serializer +from framework.forks import BaseFork +from pydantic import Field, field_serializer +from lean_spec.base import CamelModel +from lean_spec.config import LEAN_ENV -class BaseConsensusFixture(BaseFixture): + +class BaseConsensusFixture(CamelModel): """ Base class for all consensus test fixtures. - Inherits shared functionality from framework.fixtures.BaseFixture - and adds consensus-specific behavior if needed. + Provides: + - JSON serialization with custom encoders + - Hash generation for fixtures + - Common metadata handling """ - # Class-level registry of all consensus fixture formats - # Override parent's formats to maintain a separate registry - formats: ClassVar[dict[str, type["BaseConsensusFixture"]]] = {} + # Fixture format metadata + format_name: ClassVar[str] = "" + """The name of this fixture format (e.g., 'state_transition_test').""" + + description: ClassVar[str] = "Unknown fixture format" + """Human-readable description of what this fixture tests.""" + + # Instance fields + network: str | None = None + """The fork/network this fixture is valid for (e.g., 'Devnet', 'Shanghai').""" + + lean_env: str = Field(default=LEAN_ENV) + """The target lean environment (e.g. 'test' or 'prod').""" + + info: dict[str, Any] = Field(default_factory=dict, alias="_info") + """Metadata about the test (description, fork, etc.).""" expect_exception: type[Exception] | None = None """ @@ -26,18 +47,6 @@ class BaseConsensusFixture(BaseFixture): The test passes only if the exception is raised. """ - @classmethod - def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: - """ - Auto-register consensus fixture formats when subclasses are defined. - - Overrides parent to register in BaseConsensusFixture.formats instead - of BaseFixture.formats. - """ - super().__pydantic_init_subclass__(**kwargs) - if cls.format_name: - BaseConsensusFixture.formats[cls.format_name] = cls - @field_serializer("expect_exception", when_used="json") def serialize_exception(self, exception_type: type[Exception] | None) -> str | None: """Serialize exception type to its class name for JSON output.""" @@ -73,3 +82,69 @@ def assert_expected_outcome(self, exception_raised: Exception | None) -> None: f"Expected {self.expect_exception.__name__} but got " f"{type(exception_raised).__name__}: {exception_raised}" ) + + @cached_property + def json_dict(self) -> dict[str, Any]: + """ + Return the JSON representation of the fixture. + + Excludes the `info` field and converts snake_case to camelCase. + """ + return self.to_json( + exclude_none=True, + exclude={"info"}, + ) + + @cached_property + def hash(self) -> str: + """ + Generate a deterministic hash for this fixture. + + The hash is computed from the JSON representation to ensure + consistency across runs. + """ + json_str = json.dumps( + self.json_dict, + sort_keys=True, + separators=(",", ":"), + ) + h = hashlib.sha256(json_str.encode("utf-8")).hexdigest() + return f"0x{h}" + + def json_dict_with_info(self, hash_only: bool = False) -> dict[str, Any]: + """ + Return JSON representation with the info field included. + + Args: + hash_only: If True, only include the hash in _info. + + Returns: + Dictionary ready for JSON serialization. + """ + dict_with_info = self.json_dict.copy() + dict_with_info["_info"] = {"hash": self.hash} + if not hash_only: + dict_with_info["_info"].update(self.info) + return dict_with_info + + def fill_info( + self, + test_id: str, + description: str, + fork: BaseFork, + ) -> None: + """ + Fill metadata information for this fixture. + + Args: + test_id: Unique identifier for the test case. + description: Human-readable description of the test. + fork: The fork this test is valid for. + """ + if "comment" not in self.info: + self.info["comment"] = "`leanSpec` generated test" + self.info["testId"] = test_id + self.info["description"] = description + self.info["fixtureFormat"] = self.format_name + # Set network field on the fixture itself + self.network = fork.name() diff --git a/packages/testing/src/framework/__init__.py b/packages/testing/src/framework/__init__.py index 9addb6831..0f87f9972 100644 --- a/packages/testing/src/framework/__init__.py +++ b/packages/testing/src/framework/__init__.py @@ -1,6 +1 @@ -""" -Shared testing infrastructure for Ethereum consensus and execution layers. - -This module provides base classes and utilities that are common across -both consensus and execution layer testing. -""" +"""Shared testing infrastructure for Lean Ethereum spec tests.""" diff --git a/packages/testing/src/framework/cli/fill.py b/packages/testing/src/framework/cli/fill.py index a3f2f8758..96a51328b 100644 --- a/packages/testing/src/framework/cli/fill.py +++ b/packages/testing/src/framework/cli/fill.py @@ -1,4 +1,4 @@ -"""Unified CLI command for generating Ethereum test fixtures across all layers.""" +"""CLI command for generating Lean Ethereum consensus test fixtures.""" import os import sys @@ -25,13 +25,7 @@ @click.option( "--fork", required=True, - help="Fork to generate fixtures for (e.g., Lstar for consensus)", -) -@click.option( - "--layer", - type=click.Choice(["consensus", "execution"], case_sensitive=False), - default="consensus", - help="Ethereum layer to generate fixtures for (default: consensus)", + help="Fork to generate fixtures for (e.g., Lstar)", ) @click.option( "--clean", @@ -50,21 +44,14 @@ def fill( pytest_args: Sequence[str], output: str, fork: str, - layer: str, clean: bool, scheme: str, ) -> None: """ - Generate Ethereum test fixtures from test specifications. - - This unified command works across both consensus and execution layers. - The --layer flag determines which layer's forks and fixtures to use. + Generate consensus test fixtures from test specifications. Examples: - # Generate consensus layer fixtures - fill tests/consensus/devnet --fork=Lstar --layer=consensus --clean -v - - # Default layer is consensus + # Generate consensus fixtures fill tests/consensus/devnet --fork=Lstar --clean -v # Use specific XMSS scheme (overrides LEAN_ENV env var) @@ -75,17 +62,16 @@ def fill( # environment. os.environ["LEAN_ENV"] = scheme.lower() - # Check and download keys if needed (only for consensus layer) - if layer.lower() == "consensus": - # Import here to avoid loading leanSpec modules before LEAN_ENV is set - from consensus_testing.keys import download_keys, get_keys_directory + # Check and download keys if needed + # Import here to avoid loading leanSpec modules before LEAN_ENV is set + from consensus_testing.keys import download_keys, get_keys_directory - keys_directory = get_keys_directory(scheme.lower()) + keys_directory = get_keys_directory(scheme.lower()) - # Check if keys already exist, if not, download them - if not (keys_directory.exists() and any(keys_directory.glob("*.json"))): - click.echo(f"Test keys for '{scheme}' scheme not found. Downloading...") - download_keys(scheme.lower()) + # Check if keys already exist, if not, download them + if not (keys_directory.exists() and any(keys_directory.glob("*.json"))): + click.echo(f"Test keys for '{scheme}' scheme not found. Downloading...") + download_keys(scheme.lower()) config_path = Path(__file__).parent / "pytest_ini_files" / "pytest-fill.ini" # Find project root by looking for pyproject.toml with [tool.uv.workspace] @@ -105,7 +91,6 @@ def fill( f"--rootdir={project_root}", f"--output={output}", f"--fork={fork}", - f"--layer={layer}", ] if clean: diff --git a/packages/testing/src/framework/cli/pytest_ini_files/pytest-fill.ini b/packages/testing/src/framework/cli/pytest_ini_files/pytest-fill.ini index 0275c2207..c35503c9b 100644 --- a/packages/testing/src/framework/cli/pytest_ini_files/pytest-fill.ini +++ b/packages/testing/src/framework/cli/pytest_ini_files/pytest-fill.ini @@ -1,9 +1,8 @@ [pytest] # Configuration for fill command -# Search for layer-specific tests -# The actual testpath will be determined dynamically by the --layer flag -# in the pytest plugin +# Search for spec tests +# The pytest plugin restricts collection to the consensus spec tests testpaths = tests # Load pytest plugins diff --git a/packages/testing/src/framework/pytest_plugins/filler.py b/packages/testing/src/framework/pytest_plugins/filler.py index e8cc180a4..9dd856432 100644 --- a/packages/testing/src/framework/pytest_plugins/filler.py +++ b/packages/testing/src/framework/pytest_plugins/filler.py @@ -1,32 +1,45 @@ -"""Layer-agnostic pytest plugin for generating Ethereum test fixtures.""" +"""Pytest plugin for generating Lean Ethereum consensus test fixtures.""" -import importlib import json import shutil import sys from collections import defaultdict -from collections.abc import Callable from pathlib import Path from typing import Any import pytest +from consensus_testing import generate_pre_state +from consensus_testing.forks import registry +from consensus_testing.test_fixtures import ( + ApiEndpointTest, + ForkChoiceTest, + GossipsubHandlerTest, + JustifiabilityTest, + NetworkingCodecTest, + PoseidonPermutationTest, + SlotClockTest, + SSZTest, + StateTransitionTest, + SyncTest, + VerifyMultiMessageProofsTest, + VerifySignaturesTest, + VerifySingleMessageProofsTest, +) class FixtureCollector: """Collects generated fixtures and writes them to disk.""" - def __init__(self, output_directory: Path, fork: str, layer: str): + def __init__(self, output_directory: Path, fork: str): """ Initialize the fixture collector. Args: output_directory: Root directory for generated fixtures. fork: The fork name (e.g., "Lstar"). - layer: The Ethereum layer (e.g., "consensus", "execution"). """ self.output_directory = output_directory self.fork = fork - self.layer = layer self.fixtures: list[tuple[str, str, Any, str]] = [] def add_fixture( @@ -56,21 +69,19 @@ def add_fixture( base_func_name = func_name_with_params.split("[")[0] test_file = Path(test_file_path) - # Extract test path relative to tests/{layer} + # Extract test path relative to the consensus spec tests # e.g., tests/consensus/lstar/... -> lstar/... - layer = config.test_layer if hasattr(config, "test_layer") else "consensus" - try: - relative_path = test_file.relative_to(f"tests/{layer}") + relative_path = test_file.relative_to("tests/consensus") except ValueError: # Fallback: try to extract from full path relative_path = test_file test_path = relative_path.with_suffix("") - # Build output path: fixtures/{layer}/{format}/{test_path} + # Build output path: fixtures/consensus/{format}/{test_path} format_directory = fixture_format.replace("_test", "") - fixture_directory = self.output_directory / layer / format_directory / test_path + fixture_directory = self.output_directory / "consensus" / format_directory / test_path fixture_path = fixture_directory / f"{base_func_name}.json" config.fixture_path_absolute = str(fixture_path.absolute()) # type: ignore[attribute-defined] @@ -93,19 +104,19 @@ def write_fixtures(self) -> None: for (test_file_path, base_func_name, fixture_format), fixtures_list in grouped.items(): test_file = Path(test_file_path) - # Extract test path relative to tests/{layer} + # Extract test path relative to the consensus spec tests # e.g., tests/consensus/lstar/... -> lstar/... try: - relative_path = test_file.relative_to(f"tests/{self.layer}") + relative_path = test_file.relative_to("tests/consensus") except ValueError: # Fallback: use full path relative_path = test_file test_path = relative_path.with_suffix("") - # Build output path: fixtures/{layer}/{format}/{test_path} + # Build output path: fixtures/consensus/{format}/{test_path} format_directory = fixture_format.replace("_test", "") - fixture_directory = self.output_directory / self.layer / format_directory / test_path + fixture_directory = self.output_directory / "consensus" / format_directory / test_path fixture_directory.mkdir(parents=True, exist_ok=True) output_file = fixture_directory / f"{base_func_name}.json" @@ -135,12 +146,6 @@ def pytest_addoption(parser: pytest.Parser) -> None: required=True, help="Fork to generate fixtures for", ) - group.addoption( - "--layer", - action="store", - default="consensus", - help="Ethereum layer (consensus or execution, default: consensus)", - ) group.addoption( "--clean", action="store_true", @@ -149,18 +154,13 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) -def pytest_ignore_collect(collection_path: Path, config: pytest.Config) -> bool | None: +def pytest_ignore_collect(collection_path: Path) -> bool | None: """ - Ignore test collection for paths not in the current layer. + Ignore test collection for paths outside the consensus spec tests. - This prevents pytest from collecting tests from other layers, + This prevents pytest from collecting unit tests during fill, reducing overhead significantly when there are many tests. """ - if not hasattr(config, "test_layer"): - return None - - layer = config.test_layer - # Check if path is under tests/ directory try: relative_path = collection_path.relative_to(Path.cwd() / "tests") @@ -168,51 +168,19 @@ def pytest_ignore_collect(collection_path: Path, config: pytest.Config) -> bool # Not under tests/, let pytest handle it normally return None - # If it's directly under tests/{layer}, don't ignore - if str(relative_path).startswith(layer): + # If it's directly under tests/consensus, don't ignore + if str(relative_path).startswith("consensus"): return None - # Check if it's a different layer directory or unit tests - path_components = relative_path.parts - if path_components: - # Known layer directories - known_layers = {"consensus", "execution"} - if path_components[0] in known_layers: - # It's a different layer, ignore it - return True - # It's probably unit tests (tests/lean_spec), ignore during fill + # Anything else under tests/ (unit, api, interop tests) is skipped during fill + if relative_path.parts: return True return None def pytest_configure(config: pytest.Config) -> None: - """Setup fixture generation session with layer-specific modules.""" - # Get layer and validate - layer = config.getoption("--layer", default="consensus").lower() - known_layers = {"consensus", "execution"} - if layer not in known_layers: - pytest.exit( - f"Invalid layer: {layer}. Must be one of: {', '.join(known_layers)}", - returncode=pytest.ExitCode.USAGE_ERROR, - ) - - # Store layer for later use (needed by pytest_ignore_collect hook) - config.test_layer = layer # type: ignore[attribute-defined] - - # Dynamically import layer-specific package - try: - layer_module = importlib.import_module(f"{layer}_testing") - config.layer_module = layer_module # type: ignore[attribute-defined] - except ImportError as exception: - pytest.exit( - f"Failed to import {layer}_testing module: {exception}", - returncode=pytest.ExitCode.USAGE_ERROR, - ) - - # Register layer-specific test fixture formats - _register_layer_fixtures(config, layer) - + """Setup the fixture generation session.""" # Register fork validity markers config.addinivalue_line( "markers", @@ -232,15 +200,13 @@ def pytest_configure(config: pytest.Config) -> None: fork_name = config.getoption("--fork") clean = config.getoption("--clean") - # Get available forks from layer-specific module - registry = layer_module.forks.registry available_fork_names = sorted(fork.name() for fork in registry.forks) # Validate fork if not fork_name: print("Error: --fork is required", file=sys.stderr) print( - f"Available {layer} forks: {', '.join(available_fork_names)}", + f"Available forks: {', '.join(available_fork_names)}", file=sys.stderr, ) pytest.exit("Missing required --fork option.", returncode=pytest.ExitCode.USAGE_ERROR) @@ -248,11 +214,11 @@ def pytest_configure(config: pytest.Config) -> None: fork_class = registry.get_fork_by_name(fork_name) if fork_class is None: print( - f"Error: Unsupported fork for {layer} layer: {fork_name}\n", + f"Error: Unsupported fork: {fork_name}\n", file=sys.stderr, ) print( - f"Available {layer} forks: {', '.join(available_fork_names)}", + f"Available forks: {', '.join(available_fork_names)}", file=sys.stderr, ) pytest.exit("Invalid fork specified.", returncode=pytest.ExitCode.USAGE_ERROR) @@ -276,8 +242,7 @@ def pytest_configure(config: pytest.Config) -> None: output_directory.mkdir(parents=True, exist_ok=True) - # Create collector with layer info - config.fixture_collector = FixtureCollector(output_directory, fork_name, layer) # type: ignore[attribute-defined] + config.fixture_collector = FixtureCollector(output_directory, fork_name) # type: ignore[attribute-defined] config.test_fork_class = fork_class # type: ignore[attribute-defined] @@ -287,14 +252,12 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item return fork_class = config.test_fork_class - layer_module = config.layer_module - registry = layer_module.forks.registry verbose = config.getoption("verbose") deselected_items = [] selected_items = [] for test_item in items: - if not _is_test_item_valid_for_fork(test_item, fork_class, registry.get_fork_by_name): + if not _check_markers_valid_for_fork(list(test_item.iter_markers()), fork_class): if verbose < 2: deselected_items.append(test_item) else: @@ -310,7 +273,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item def _check_markers_valid_for_fork( markers: list[Any], fork_class: type, - get_fork_by_name: Callable[[str], type | None], ) -> bool: """ Check if test markers indicate validity for the given fork. @@ -329,19 +291,19 @@ def _check_markers_valid_for_fork( if marker.name == "valid_from": has_valid_from = True for fork_name in marker.args: - target_fork = get_fork_by_name(fork_name) + target_fork = registry.get_fork_by_name(fork_name) if target_fork: valid_from_forks.append(target_fork) elif marker.name == "valid_until": has_valid_until = True for fork_name in marker.args: - target_fork = get_fork_by_name(fork_name) + target_fork = registry.get_fork_by_name(fork_name) if target_fork: valid_until_forks.append(target_fork) elif marker.name == "valid_at": has_valid_at = True for fork_name in marker.args: - target_fork = get_fork_by_name(fork_name) + target_fork = registry.get_fork_by_name(fork_name) if target_fork: valid_at_forks.append(target_fork) @@ -362,15 +324,6 @@ def _check_markers_valid_for_fork( return from_valid and until_valid -def _is_test_item_valid_for_fork( - item: pytest.Item, - fork_class: type, - get_fork_by_name: Callable[[str], type | None], -) -> bool: - """Check if a test item is valid for the given fork based on validity markers.""" - return _check_markers_valid_for_fork(list(item.iter_markers()), fork_class, get_fork_by_name) - - def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: """Write all collected fixtures at the end of the session.""" if hasattr(session.config, "fixture_collector"): @@ -427,26 +380,17 @@ def test_case_description(request: pytest.FixtureRequest) -> str: @pytest.fixture(scope="function") def pre(request: pytest.FixtureRequest, fork: Any) -> Any: """ - Default pre-state (layer-specific). + Default consensus pre-state. Tests can request this fixture to customize the initial state, or omit it to use the default (auto-injected by framework). """ - layer = request.config.test_layer # type: ignore[attribute-defined] - - if layer == "execution": - pytest.exit( - "Execution layer testing is not yet implemented. Use --layer=consensus (default).", - returncode=pytest.ExitCode.USAGE_ERROR, - ) - - layer_module = request.config.layer_module # type: ignore[attribute-defined] spec = fork.spec_class()() if hasattr(request, "param"): - return layer_module.generate_pre_state(fork=spec, **request.param) + return generate_pre_state(fork=spec, **request.param) - return layer_module.generate_pre_state(fork=spec) + return generate_pre_state(fork=spec) def base_spec_filler_parametrizer(fixture_class: Any) -> Any: @@ -514,10 +458,8 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: return fork_class = metafunc.config.test_fork_class # type: ignore[attribute-defined] - layer_module = metafunc.config.layer_module # type: ignore[attribute-defined] - registry = layer_module.forks.registry - if not _is_test_valid_for_fork(metafunc, fork_class, registry.get_fork_by_name): + if not _check_markers_valid_for_fork(list(metafunc.definition.iter_markers()), fork_class): verbose = metafunc.config.getoption("verbose") if verbose >= 2: metafunc.parametrize( @@ -541,37 +483,18 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: ) -def _is_test_valid_for_fork( - metafunc: pytest.Metafunc, fork_class: Any, get_fork_by_name: Any -) -> bool: - """Check if a test is valid for the given fork based on validity markers.""" - return _check_markers_valid_for_fork( - list(metafunc.definition.iter_markers()), fork_class, get_fork_by_name - ) - - -def _register_layer_fixtures(config: pytest.Config, layer: str) -> None: - """Register layer-specific test fixture formats during configuration.""" - try: - # Import the test_fixtures module - fixtures_module = importlib.import_module(f"{layer}_testing.test_fixtures") - - # Get the base fixture class based on layer - if layer == "consensus": - base_fixture_class = fixtures_module.BaseConsensusFixture - elif layer == "execution": - base_fixture_class = fixtures_module.BaseExecutionFixture - else: - return - - # Register all fixture formats globally so pytest can discover them - # This must happen during pytest_configure, before fixture discovery - for format_name, fixture_class in base_fixture_class.formats.items(): - fixture_func = base_spec_filler_parametrizer(fixture_class) - # Add to module globals so pytest can discover them - globals()[format_name] = fixture_func - except (ImportError, AttributeError) as exception: - pytest.exit( - f"Failed to load {layer} layer test fixtures: {exception}", - returncode=pytest.ExitCode.USAGE_ERROR, - ) +# Pytest fixtures for every consensus fixture format. +# Each spec test requests one by its format name and calls it to build a test vector. +api_endpoint = base_spec_filler_parametrizer(ApiEndpointTest) +fork_choice_test = base_spec_filler_parametrizer(ForkChoiceTest) +gossipsub_handler = base_spec_filler_parametrizer(GossipsubHandlerTest) +justifiability = base_spec_filler_parametrizer(JustifiabilityTest) +networking_codec = base_spec_filler_parametrizer(NetworkingCodecTest) +poseidon_permutation = base_spec_filler_parametrizer(PoseidonPermutationTest) +slot_clock = base_spec_filler_parametrizer(SlotClockTest) +ssz = base_spec_filler_parametrizer(SSZTest) +state_transition_test = base_spec_filler_parametrizer(StateTransitionTest) +sync = base_spec_filler_parametrizer(SyncTest) +verify_multi_message_proofs_test = base_spec_filler_parametrizer(VerifyMultiMessageProofsTest) +verify_signatures_test = base_spec_filler_parametrizer(VerifySignaturesTest) +verify_single_message_proofs_test = base_spec_filler_parametrizer(VerifySingleMessageProofsTest) diff --git a/packages/testing/src/framework/test_fixtures/__init__.py b/packages/testing/src/framework/test_fixtures/__init__.py deleted file mode 100644 index 45ad69b4b..000000000 --- a/packages/testing/src/framework/test_fixtures/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Base fixture infrastructure for Ethereum testing.""" - -from framework.test_fixtures.base import BaseFixture - -__all__ = [ - "BaseFixture", -] diff --git a/packages/testing/src/framework/test_fixtures/base.py b/packages/testing/src/framework/test_fixtures/base.py deleted file mode 100644 index cf3b01c62..000000000 --- a/packages/testing/src/framework/test_fixtures/base.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Base fixture definitions for Ethereum test formats.""" - -import hashlib -import json -from functools import cached_property -from typing import Any, ClassVar - -from pydantic import Field - -from framework.forks import BaseFork -from lean_spec.base import CamelModel -from lean_spec.config import LEAN_ENV - - -class BaseFixture(CamelModel): - """ - Base class for all Ethereum test fixtures (consensus and execution layers). - - Provides: - - Auto-registration of fixture formats - - JSON serialization with custom encoders - - Hash generation for fixtures - - Common metadata handling - - This base class is layer-agnostic and can be used for both consensus - and execution layer fixtures. - """ - - # Class-level registry of all fixture formats - formats: ClassVar[dict[str, type["BaseFixture"]]] = {} - - # Fixture format metadata - format_name: ClassVar[str] = "" - """The name of this fixture format (e.g., 'state_transition_test').""" - - description: ClassVar[str] = "Unknown fixture format" - """Human-readable description of what this fixture tests.""" - - # Instance fields - network: str | None = None - """The fork/network this fixture is valid for (e.g., 'Devnet', 'Shanghai').""" - - lean_env: str = Field(default=LEAN_ENV) - """The target lean environment (e.g. 'test' or 'prod').""" - - info: dict[str, Any] = Field(default_factory=dict, alias="_info") - """Metadata about the test (description, fork, etc.).""" - - @classmethod - def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: - """ - Auto-register fixture formats when subclasses are defined. - - This hook is called automatically when a new subclass is created. - Registration is a no-op here; layer base classes (e.g. BaseConsensusFixture) - override this to register into their own separate registry. - """ - super().__pydantic_init_subclass__(**kwargs) - - @cached_property - def json_dict(self) -> dict[str, Any]: - """ - Return the JSON representation of the fixture. - - Excludes the `info` field and converts snake_case to camelCase. - """ - return self.to_json( - exclude_none=True, - exclude={"info"}, - ) - - @cached_property - def hash(self) -> str: - """ - Generate a deterministic hash for this fixture. - - The hash is computed from the JSON representation to ensure - consistency across runs. - """ - json_str = json.dumps( - self.json_dict, - sort_keys=True, - separators=(",", ":"), - ) - h = hashlib.sha256(json_str.encode("utf-8")).hexdigest() - return f"0x{h}" - - def json_dict_with_info(self, hash_only: bool = False) -> dict[str, Any]: - """ - Return JSON representation with the info field included. - - Args: - hash_only: If True, only include the hash in _info. - - Returns: - Dictionary ready for JSON serialization. - """ - dict_with_info = self.json_dict.copy() - dict_with_info["_info"] = {"hash": self.hash} - if not hash_only: - dict_with_info["_info"].update(self.info) - return dict_with_info - - def fill_info( - self, - test_id: str, - description: str, - fork: BaseFork, - ) -> None: - """ - Fill metadata information for this fixture. - - Args: - test_id: Unique identifier for the test case. - description: Human-readable description of the test. - fork: The fork this test is valid for. - """ - if "comment" not in self.info: - self.info["comment"] = "`leanSpec` generated test" - self.info["testId"] = test_id - self.info["description"] = description - self.info["fixtureFormat"] = self.format_name - - # Set network field on the fixture itself - self.network = fork.name()