diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 083866d5a6c..0948d37fc5e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,7 @@ Test fixtures for use by clients are available for each release on the [Github r - 🔀 Make `BaseFixture` able to parse any fixture format such as `BlockchainFixture` ([#1210](https://github.com/ethereum/execution-spec-tests/pull/1210)). - ✨ Blockchain and Blockchain-Engine tests now have a marker to specify that they were generated from a state test, which can be used with `-m blockchain_test_from_state_test` and `-m blockchain_test_engine_from_state_test` respectively ([#1220](https://github.com/ethereum/execution-spec-tests/pull/1220)). - ✨ Blockchain and Blockchain-Engine tests that were generated from a state test now have `blockchain_test_from_state_test` or `blockchain_test_engine_from_state_test` as part of their test IDs ([#1220](https://github.com/ethereum/execution-spec-tests/pull/1220)). +- 🔀 Refactor `ethereum_test_fixtures` and `ethereum_clis` to create `FixtureConsumer` and `FixtureConsumerTool` classes which abstract away the consumption process used by `consume direct` ([#935](https://github.com/ethereum/execution-spec-tests/pull/935)). ### 📋 Misc diff --git a/src/ethereum_clis/__init__.py b/src/ethereum_clis/__init__.py index a8d3c0ef65e..76f5cfd6990 100644 --- a/src/ethereum_clis/__init__.py +++ b/src/ethereum_clis/__init__.py @@ -4,25 +4,29 @@ from .clis.ethereumjs import EthereumJSTransitionTool from .clis.evmone import EvmoneExceptionMapper, EvmOneTransitionTool from .clis.execution_specs import ExecutionSpecsTransitionTool -from .clis.geth import GethTransitionTool +from .clis.geth import GethFixtureConsumer, GethTransitionTool from .clis.nimbus import NimbusTransitionTool from .ethereum_cli import CLINotFoundInPathError, UnknownCLIError +from .fixture_consumer_tool import FixtureConsumerTool from .transition_tool import TransitionTool from .types import Result, TransitionToolOutput TransitionTool.set_default_tool(ExecutionSpecsTransitionTool) +FixtureConsumerTool.set_default_tool(GethFixtureConsumer) __all__ = ( "BesuTransitionTool", + "CLINotFoundInPathError", "EthereumJSTransitionTool", + "EvmoneExceptionMapper", "EvmOneTransitionTool", "ExecutionSpecsTransitionTool", + "FixtureConsumerTool", + "GethFixtureConsumer", "GethTransitionTool", - "EvmoneExceptionMapper", "NimbusTransitionTool", "Result", "TransitionTool", "TransitionToolOutput", - "CLINotFoundInPathError", "UnknownCLIError", ) diff --git a/src/ethereum_clis/clis/geth.py b/src/ethereum_clis/clis/geth.py index 13d86782afc..0d96b0ffc39 100644 --- a/src/ethereum_clis/clis/geth.py +++ b/src/ethereum_clis/clis/geth.py @@ -5,8 +5,9 @@ import shutil import subprocess import textwrap +from functools import cache from pathlib import Path -from typing import Optional +from typing import Any, Dict, List, Optional from ethereum_test_exceptions import ( EOFException, @@ -14,140 +15,12 @@ ExceptionMessage, TransactionException, ) -from ethereum_test_fixtures import BlockchainFixture, StateFixture +from ethereum_test_fixtures import BlockchainFixture, FixtureFormat, StateFixture from ethereum_test_forks import Fork -from ..transition_tool import FixtureFormat, TransitionTool, dump_files_to_directory - - -class GethTransitionTool(TransitionTool): - """Go-ethereum `evm` Transition tool interface wrapper class.""" - - default_binary = Path("evm") - detect_binary_pattern = re.compile(r"^evm(.exe)? version\b") - t8n_subcommand: Optional[str] = "t8n" - statetest_subcommand: Optional[str] = "statetest" - blocktest_subcommand: Optional[str] = "blocktest" - binary: Path - cached_version: Optional[str] = None - trace: bool - t8n_use_stream = True - - def __init__( - self, - *, - binary: Optional[Path] = None, - trace: bool = False, - ): - """Initialize the Go-ethereum Transition tool interface.""" - super().__init__(exception_mapper=GethExceptionMapper(), binary=binary, trace=trace) - args = [str(self.binary), str(self.t8n_subcommand), "--help"] - try: - result = subprocess.run(args, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - raise Exception( - f"evm process unexpectedly returned a non-zero status code: {e}." - ) from e - except Exception as e: - raise Exception(f"Unexpected exception calling evm tool: {e}.") from e - self.help_string = result.stdout - - def is_fork_supported(self, fork: Fork) -> bool: - """ - Return True if the fork is supported by the tool. - - If the fork is a transition fork, we want to check the fork it transitions to. - """ - return fork.transition_tool_name() in self.help_string - - def get_blocktest_help(self) -> str: - """Return the help string for the blocktest subcommand.""" - args = [str(self.binary), "blocktest", "--help"] - try: - result = subprocess.run(args, capture_output=True, text=True) - except subprocess.CalledProcessError as e: - raise Exception( - f"evm process unexpectedly returned a non-zero status code: {e}." - ) from e - except Exception as e: - raise Exception(f"Unexpected exception calling evm tool: {e}.") from e - return result.stdout - - def is_verifiable( - self, - fixture_format: FixtureFormat, - ) -> bool: - """Return whether the fixture format is verifiable by this Geth's evm tool.""" - return fixture_format in {StateFixture, BlockchainFixture} - - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ): - """Execute `evm [state|block]test` to verify the fixture at `fixture_path`.""" - command: list[str] = [str(self.binary)] - - if debug_output_path: - command += ["--debug", "--json", "--verbosity", "100"] - - if fixture_format == StateFixture: - assert self.statetest_subcommand, "statetest subcommand not set" - command.append(self.statetest_subcommand) - elif fixture_format == BlockchainFixture: - assert self.blocktest_subcommand, "blocktest subcommand not set" - command.append(self.blocktest_subcommand) - else: - raise Exception(f"Invalid test fixture format: {fixture_format}") - - if fixture_name and fixture_format == BlockchainFixture: - assert isinstance(fixture_name, str), "fixture_name must be a string" - command.append("--run") - command.append(fixture_name) - command.append(str(fixture_path)) - - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - if debug_output_path: - debug_fixture_path = debug_output_path / "fixtures.json" - # Use the local copy of the fixture in the debug directory - verify_fixtures_call = " ".join(command[:-1]) + f" {debug_fixture_path}" - verify_fixtures_script = textwrap.dedent( - f"""\ - #!/bin/bash - {verify_fixtures_call} - """ - ) - dump_files_to_directory( - str(debug_output_path), - { - "verify_fixtures_args.py": command, - "verify_fixtures_returncode.txt": result.returncode, - "verify_fixtures_stdout.txt": result.stdout.decode(), - "verify_fixtures_stderr.txt": result.stderr.decode(), - "verify_fixtures.sh+x": verify_fixtures_script, - }, - ) - shutil.copyfile(fixture_path, debug_fixture_path) - - if result.returncode != 0: - raise Exception( - f"EVM test failed.\n{' '.join(command)}\n\n Error:\n{result.stderr.decode()}" - ) - - if fixture_format == StateFixture: - result_json = json.loads(result.stdout.decode()) - if not isinstance(result_json, list): - raise Exception(f"Unexpected result from evm statetest: {result_json}") - else: - result_json = [] # there is no parseable format for blocktest output - return result_json +from ..ethereum_cli import EthereumCLI +from ..fixture_consumer_tool import FixtureConsumerTool +from ..transition_tool import TransitionTool, dump_files_to_directory class GethExceptionMapper(ExceptionMapper): @@ -280,3 +153,239 @@ def _mapping_data(self): EOFException.INVALID_CODE_SECTION_INDEX, "err: invalid_code_section_index" ), ] + + +class GethEvm(EthereumCLI): + """go-ethereum `evm` base class.""" + + default_binary = Path("evm") + detect_binary_pattern = re.compile(r"^evm(.exe)? version\b") + cached_version: Optional[str] = None + + def __init__( + self, + binary: Path, + trace: bool = False, + exception_mapper: ExceptionMapper | None = None, + ): + """Initialize the GethEvm class.""" + self.binary = binary + self.trace = trace + self.exception_mapper = exception_mapper if exception_mapper else GethExceptionMapper() + + def _run_command(self, command: List[str]) -> subprocess.CompletedProcess: + try: + return subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + except subprocess.CalledProcessError as e: + raise Exception("Command failed with non-zero status.") from e + except Exception as e: + raise Exception("Unexpected exception calling evm tool.") from e + + def _consume_debug_dump( + self, + command: List[str], + result: subprocess.CompletedProcess, + fixture_path: Path, + debug_output_path: Path, + ): + debug_fixture_path = debug_output_path / "fixtures.json" + consume_direct_call = " ".join(command[:-1]) + f" {debug_fixture_path}" + consume_direct_script = textwrap.dedent( + f"""\ + #!/bin/bash + {consume_direct_call} + """ + ) + dump_files_to_directory( + str(debug_output_path), + { + "consume_direct_args.py": command, + "consume_direct_returncode.txt": result.returncode, + "consume_direct_stdout.txt": result.stdout, + "consume_direct_stderr.txt": result.stderr, + "consume_direct.sh+x": consume_direct_script, + }, + ) + shutil.copyfile(fixture_path, debug_fixture_path) + + @cache # noqa + def help(self, subcommand: str | None = None) -> str: + """Return the help string, optionally for a subcommand.""" + help_command = [str(self.binary)] + if subcommand: + help_command.append(subcommand) + help_command.append("--help") + return self._run_command(help_command).stdout + + +class GethTransitionTool(GethEvm, TransitionTool): + """go-ethereum `evm` Transition tool interface wrapper class.""" + + subcommand: Optional[str] = "t8n" + trace: bool + t8n_use_stream = True + + def __init__(self, *, binary: Path, trace: bool = False): + """Initialize the GethTransitionTool class.""" + super().__init__(binary=binary, trace=trace) + help_command = [str(self.binary), str(self.subcommand), "--help"] + result = self._run_command(help_command) + self.help_string = result.stdout + + def is_fork_supported(self, fork: Fork) -> bool: + """ + Return True if the fork is supported by the tool. + + If the fork is a transition fork, we want to check the fork it transitions to. + """ + return fork.transition_tool_name() in self.help_string + + +class GethFixtureConsumer( + GethEvm, + FixtureConsumerTool, + fixture_formats=[StateFixture, BlockchainFixture], +): + """Geth's implementation of the fixture consumer.""" + + def consume_blockchain_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ): + """ + Consume a single blockchain test. + + The `evm blocktest` command takes the `--run` argument which can be used to select a + specific fixture from the fixture file when executing. + """ + subcommand = "blocktest" + global_options = [] + subcommand_options = [] + if debug_output_path: + global_options += ["--verbosity", "100"] + subcommand_options += ["--trace"] + + if fixture_name: + subcommand_options += ["--run", re.escape(fixture_name)] + + command = ( + [str(self.binary)] + + global_options + + [subcommand] + + subcommand_options + + [str(fixture_path)] + ) + + result = self._run_command(command) + + if debug_output_path: + self._consume_debug_dump(command, result, fixture_path, debug_output_path) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}" + ) + + @cache # noqa + def consume_state_test_file( + self, + fixture_path: Path, + debug_output_path: Optional[Path] = None, + ) -> List[Dict[str, Any]]: + """ + Consume an entire state test file. + + The `evm statetest` will always execute all the tests contained in a file without the + possibility of selecting a single test, so this function is cached in order to only call + the command once and `consume_state_test` can simply select the result that + was requested. + """ + subcommand = "statetest" + global_options: List[str] = [] + subcommand_options: List[str] = [] + if debug_output_path: + global_options += ["--verbosity", "100"] + subcommand_options += ["--trace"] + + command = ( + [str(self.binary)] + + global_options + + [subcommand] + + subcommand_options + + [str(fixture_path)] + ) + result = self._run_command(command) + + if debug_output_path: + self._consume_debug_dump(command, result, fixture_path, debug_output_path) + + if result.returncode != 0: + raise Exception( + f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}" + ) + + result_json = json.loads(result.stdout) + if not isinstance(result_json, list): + raise Exception(f"Unexpected result from evm statetest: {result_json}") + return result_json + + def consume_state_test( + self, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ): + """ + Consume a single state test. + + Uses the cached result from `consume_state_test_file` in order to not call the command + every time an select a single result from there. + """ + file_results = self.consume_state_test_file( + fixture_path=fixture_path, + debug_output_path=debug_output_path, + ) + if fixture_name: + test_result = [ + test_result for test_result in file_results if test_result["name"] == fixture_name + ] + assert len(test_result) < 2, f"Multiple test results for {fixture_name}" + assert len(test_result) == 1, f"Test result for {fixture_name} missing" + assert test_result[0]["pass"], f"State test failed: {test_result[0]['error']}" + else: + if any(not test_result["pass"] for test_result in file_results): + exception_text = "State test failed: \n" + "\n".join( + f"{test_result['name']}: " + test_result["error"] + for test_result in file_results + if not test_result["pass"] + ) + raise Exception(exception_text) + + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: Optional[str] = None, + debug_output_path: Optional[Path] = None, + ): + """Execute the appropriate geth fixture consumer for the fixture at `fixture_path`.""" + if fixture_format == BlockchainFixture: + self.consume_blockchain_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + elif fixture_format == StateFixture: + self.consume_state_test( + fixture_path=fixture_path, + fixture_name=fixture_name, + debug_output_path=debug_output_path, + ) + else: + raise Exception( + f"Fixture format {fixture_format.format_name} not supported by {self.binary}" + ) diff --git a/src/ethereum_clis/ethereum_cli.py b/src/ethereum_clis/ethereum_cli.py index a79ca65070f..4e7b184d6e1 100644 --- a/src/ethereum_clis/ethereum_cli.py +++ b/src/ethereum_clis/ethereum_cli.py @@ -3,7 +3,6 @@ import os import shutil import subprocess -from abc import ABC, abstractmethod from itertools import groupby from pathlib import Path from re import Pattern @@ -26,7 +25,7 @@ def __init__(self, message="The CLI binary was not found in the path", binary=No super().__init__(message) -class EthereumCLI(ABC): +class EthereumCLI: """ Abstract base class to help create Python interfaces to Ethereum CLIs. @@ -39,13 +38,13 @@ class EthereumCLI(ABC): registered_tools: List[Type[Any]] = [] default_tool: Optional[Type[Any]] = None + binary: Path default_binary: Path detect_binary_pattern: Pattern version_flag: str = "-v" cached_version: Optional[str] = None - @abstractmethod - def __init__(self, *, binary: Optional[Path] = None, trace: bool = False): + def __init__(self, *, binary: Optional[Path] = None): """Abstract initialization method that all subclasses must implement.""" if binary is None: binary = self.default_binary @@ -58,7 +57,6 @@ def __init__(self, *, binary: Optional[Path] = None, trace: bool = False): if not binary: raise CLINotFoundInPathError(binary=binary) self.binary = Path(binary) - self.trace = trace @classmethod def register_tool(cls, tool_subclass: Type[Any]): diff --git a/src/ethereum_clis/fixture_consumer_tool.py b/src/ethereum_clis/fixture_consumer_tool.py new file mode 100644 index 00000000000..1f958e2c139 --- /dev/null +++ b/src/ethereum_clis/fixture_consumer_tool.py @@ -0,0 +1,22 @@ +"""Fixture consumer tool abstract class.""" + +from typing import List, Type + +from ethereum_test_fixtures import FixtureConsumer, FixtureFormat + +from .ethereum_cli import EthereumCLI + + +class FixtureConsumerTool(FixtureConsumer, EthereumCLI): + """ + Fixture consumer tool abstract base class which should be inherited by all fixture consumer + tool implementations. + """ + + registered_tools: List[Type["FixtureConsumerTool"]] = [] + default_tool: Type["FixtureConsumerTool"] | None = None + + def __init_subclass__(cls, *, fixture_formats: List[FixtureFormat]): + """Register all subclasses of FixtureConsumerTool as possible tools.""" + FixtureConsumerTool.register_tool(cls) + cls.fixture_formats = fixture_formats diff --git a/src/ethereum_clis/transition_tool.py b/src/ethereum_clis/transition_tool.py index c8d8ce9b7ac..24d314048f2 100644 --- a/src/ethereum_clis/transition_tool.py +++ b/src/ethereum_clis/transition_tool.py @@ -19,7 +19,6 @@ from ethereum_test_base_types import BlobSchedule from ethereum_test_exceptions import ExceptionMapper -from ethereum_test_fixtures import FixtureFormat, FixtureVerifier from ethereum_test_forks import Fork from ethereum_test_types import Alloc, Environment, Transaction @@ -39,7 +38,7 @@ SLOW_REQUEST_TIMEOUT = 60 -class TransitionTool(EthereumCLI, FixtureVerifier): +class TransitionTool(EthereumCLI): """ Transition tool abstract base class which should be inherited by all transition tool implementations. @@ -50,12 +49,9 @@ class TransitionTool(EthereumCLI, FixtureVerifier): registered_tools: List[Type["TransitionTool"]] = [] default_tool: Optional[Type["TransitionTool"]] = None - t8n_subcommand: Optional[str] = None - statetest_subcommand: Optional[str] = None - blocktest_subcommand: Optional[str] = None + subcommand: Optional[str] = None cached_version: Optional[str] = None t8n_use_stream: bool = False - t8n_use_server: bool = False server_url: str process: Optional[subprocess.Popen] = None @@ -424,8 +420,8 @@ def construct_args_stream( ) -> List[str]: """Construct arguments for t8n interaction via streams.""" command: list[str] = [str(self.binary)] - if self.t8n_subcommand: - command.append(self.t8n_subcommand) + if self.subcommand: + command.append(self.subcommand) args = command + [ "--input.alloc=stdin", @@ -541,19 +537,3 @@ def evaluate( t8n_data=t8n_data, debug_output_path=debug_output_path, ) - - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: Optional[str] = None, - debug_output_path: Optional[Path] = None, - ): - """ - Execute `evm [state|block]test` to verify the fixture at `fixture_path`. - - Currently only implemented by geth's evm. - """ - raise NotImplementedError( - "The `verify_fixture()` function is not supported by this tool. Use geth's evm tool." - ) diff --git a/src/ethereum_test_fixtures/__init__.py b/src/ethereum_test_fixtures/__init__.py index a5e32a47225..f38b4d0b8b1 100644 --- a/src/ethereum_test_fixtures/__init__.py +++ b/src/ethereum_test_fixtures/__init__.py @@ -3,10 +3,10 @@ from .base import BaseFixture, FixtureFormat, LabeledFixtureFormat from .blockchain import BlockchainEngineFixture, BlockchainFixture, BlockchainFixtureCommon from .collector import FixtureCollector, TestInfo +from .consume import FixtureConsumer from .eof import EOFFixture from .state import StateFixture from .transaction import TransactionFixture -from .verify import FixtureVerifier __all__ = [ "BaseFixture", @@ -15,9 +15,9 @@ "BlockchainFixtureCommon", "EOFFixture", "FixtureCollector", + "FixtureConsumer", "FixtureFormat", "LabeledFixtureFormat", - "FixtureVerifier", "StateFixture", "TestInfo", "TransactionFixture", diff --git a/src/ethereum_test_fixtures/collector.py b/src/ethereum_test_fixtures/collector.py index 690c3c9cd5b..aa3790dd94d 100644 --- a/src/ethereum_test_fixtures/collector.py +++ b/src/ethereum_test_fixtures/collector.py @@ -14,8 +14,8 @@ from ethereum_test_base_types import to_json from .base import BaseFixture +from .consume import FixtureConsumer from .file import Fixtures -from .verify import FixtureVerifier def strip_test_prefix(name: str) -> str: @@ -154,21 +154,21 @@ def dump_fixtures(self) -> None: raise TypeError("All fixtures in a single file must have the same format.") fixtures.collect_into_file(fixture_path) - def verify_fixture_files(self, evm_fixture_verification: FixtureVerifier) -> None: + def verify_fixture_files(self, evm_fixture_verification: FixtureConsumer) -> None: """Run `evm [state|block]test` on each fixture.""" for fixture_path, name_fixture_dict in self.all_fixtures.items(): for _fixture_name, fixture in name_fixture_dict.items(): - if evm_fixture_verification.is_verifiable(fixture.__class__): + if evm_fixture_verification.can_consume(fixture.__class__): info = self.json_path_to_test_item[fixture_path] - verify_fixtures_dump_dir = self._get_verify_fixtures_dump_dir(info) - evm_fixture_verification.verify_fixture( + consume_direct_dump_dir = self._get_consume_direct_dump_dir(info) + evm_fixture_verification.consume_fixture( fixture.__class__, fixture_path, fixture_name=None, - debug_output_path=verify_fixtures_dump_dir, + debug_output_path=consume_direct_dump_dir, ) - def _get_verify_fixtures_dump_dir( + def _get_consume_direct_dump_dir( self, info: TestInfo, ): diff --git a/src/ethereum_test_fixtures/consume.py b/src/ethereum_test_fixtures/consume.py index dfed576c6fc..bef04fa0fc5 100644 --- a/src/ethereum_test_fixtures/consume.py +++ b/src/ethereum_test_fixtures/consume.py @@ -1,18 +1,44 @@ """Defines models for index files and consume test cases.""" import datetime +from abc import ABC, abstractmethod from pathlib import Path from typing import List, TextIO from pydantic import BaseModel, RootModel from ethereum_test_base_types import HexNumber -from ethereum_test_fixtures import FixtureFormat -from .base import BaseFixture +from .base import BaseFixture, FixtureFormat from .file import Fixtures +class FixtureConsumer(ABC): + """Abstract class for verifying Ethereum test fixtures.""" + + fixture_formats: List[FixtureFormat] + + def can_consume( + self, + fixture_format: FixtureFormat, + ) -> bool: + """Return whether the fixture format is consumable by this consumer.""" + return fixture_format in self.fixture_formats + + @abstractmethod + def consume_fixture( + self, + fixture_format: FixtureFormat, + fixture_path: Path, + fixture_name: str | None = None, + debug_output_path: Path | None = None, + ): + """Test the client with the specified fixture using its direct consumer interface.""" + raise NotImplementedError( + "The `consume_fixture()` function is not supported by this tool." + ) + + class TestCaseBase(BaseModel): """Base model for a test case used in EEST consume commands.""" diff --git a/src/ethereum_test_fixtures/verify.py b/src/ethereum_test_fixtures/verify.py deleted file mode 100644 index 7ad29f260c1..00000000000 --- a/src/ethereum_test_fixtures/verify.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Ethereum test fixture verifyer abstract class.""" - -from abc import ABC, abstractmethod -from pathlib import Path - -from .base import FixtureFormat - - -class FixtureVerifier(ABC): - """Abstract class for verifying Ethereum test fixtures.""" - - def is_verifiable( - self, - fixture_format: FixtureFormat, - ) -> bool: - """Return whether the fixture format is verifiable by this verifier.""" - return False - - @abstractmethod - def verify_fixture( - self, - fixture_format: FixtureFormat, - fixture_path: Path, - fixture_name: str | None = None, - debug_output_path: Path | None = None, - ): - """ - Execute `evm [state|block]test` to verify the fixture at `fixture_path`. - - Currently only implemented by geth's evm. - """ - raise NotImplementedError( - "The `verify_fixture()` function is not supported by this tool. Use geth's evm tool." - ) diff --git a/src/pytest_plugins/consume/consume.py b/src/pytest_plugins/consume/consume.py index 1b404ec16be..2edfbbb3da7 100644 --- a/src/pytest_plugins/consume/consume.py +++ b/src/pytest_plugins/consume/consume.py @@ -205,28 +205,21 @@ def pytest_generate_tests(metafunc): fork = metafunc.config.getoption("single_fork") metafunc.parametrize( - "test_case", + "test_case,fixture_format", ( - pytest.param(test_case, id=test_case.id) + pytest.param(test_case, test_case.format, id=test_case.id) for test_case in metafunc.config.test_cases if test_case.format in metafunc.function.fixture_format and (not fork or test_case.fork == fork) ), ) - metafunc.parametrize( - "fixture_format", - ( - pytest.param(fixture_format, id=fixture_format.format_name) - for fixture_format in metafunc.function.fixture_format - ), - ) if "client_type" in metafunc.fixturenames: client_ids = [client.name for client in metafunc.config.hive_execution_clients] metafunc.parametrize("client_type", metafunc.config.hive_execution_clients, ids=client_ids) -def pytest_collection_modifyitems(session, config, items): +def pytest_collection_modifyitems(items): """Modify collected item names to remove the test runner function from the name.""" for item in items: original_name = item.originalname diff --git a/src/pytest_plugins/consume/direct/conftest.py b/src/pytest_plugins/consume/direct/conftest.py index f5dd9435bfe..8e40368b333 100644 --- a/src/pytest_plugins/consume/direct/conftest.py +++ b/src/pytest_plugins/consume/direct/conftest.py @@ -8,11 +8,11 @@ import json import tempfile from pathlib import Path -from typing import Generator, Optional +from typing import List import pytest -from ethereum_clis import TransitionTool +from ethereum_clis.fixture_consumer_tool import FixtureConsumerTool from ethereum_test_base_types import to_json from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream from ethereum_test_fixtures.file import Fixtures @@ -26,75 +26,57 @@ def pytest_addoption(parser): # noqa: D103 ) consume_group.addoption( - "--evm-bin", - action="store", - dest="evm_bin", - type=Path, - default=Path("evm"), + "--fixture-consumer-bin", + action="append", + dest="fixture_consumer_bin", + type=list, + default=[Path("evm")], help=( - "Path to an evm executable that provides `blocktest`. Default: First 'evm' entry in " - "PATH." + "Path to a geth evm executable that provides `blocktest` or `statetest`. " + "Flag can be used multiple times to specify multiple fixture consumer binaries." + "Default: First 'evm' entry in PATH." ), ) consume_group.addoption( "--traces", action="store_true", - dest="evm_collect_traces", + dest="consumer_collect_traces", default=False, - help="Collect traces of the execution information from the transition tool.", + help="Collect traces of the execution information from the fixture consumer tool.", ) debug_group = parser.getgroup("debug", "Arguments defining debug behavior") debug_group.addoption( - "--evm-dump-dir", + "--dump-dir", action="store", dest="base_dump_dir", type=Path, default=None, - help="Path to dump the transition tool debug output.", + help="Path to dump the fixture consumer tool debug output.", ) def pytest_configure(config): # noqa: D103 - evm = TransitionTool.from_binary_path( - binary_path=config.getoption("evm_bin"), - # TODO: The verify_fixture() method doesn't currently use this option. - trace=config.getoption("evm_collect_traces"), - ) - try: - blocktest_help_string = evm.get_blocktest_help() - except NotImplementedError as e: - pytest.exit(str(e)) - config.evm = evm - config.evm_run_single_test = "--run" in blocktest_help_string - - -@pytest.fixture(autouse=True, scope="session") -def evm(request) -> Generator[TransitionTool, None, None]: - """Return interface to the evm binary that will consume tests.""" - yield request.config.evm - request.config.evm.shutdown() - - -@pytest.fixture(scope="session") -def evm_run_single_test(request) -> bool: - """Specify whether to execute one test per fixture in each json file.""" - return request.config.evm_run_single_test + fixture_consumers = [] + for fixture_consumer_bin_path in config.getoption("fixture_consumer_bin"): + fixture_consumers.append( + FixtureConsumerTool.from_binary_path( + binary_path=Path(fixture_consumer_bin_path), + trace=config.getoption("consumer_collect_traces"), + ) + ) + config.fixture_consumers = fixture_consumers @pytest.fixture(scope="function") -def test_dump_dir( - request, fixture_path: Path, fixture_name: str, evm_run_single_test: bool -) -> Optional[Path]: +def test_dump_dir(request, fixture_path: Path, fixture_name: str) -> Path | None: """The directory to write evm debug output to.""" base_dump_dir = request.config.getoption("base_dump_dir") if not base_dump_dir: return None - if evm_run_single_test: - if len(fixture_name) > 142: - # ensure file name is not too long for eCryptFS - fixture_name = fixture_name[:70] + "..." + fixture_name[-70:] - return base_dump_dir / fixture_path.stem / fixture_name.replace("/", "-") - return base_dump_dir / fixture_path.stem + if len(fixture_name) > 142: + # ensure file name is not too long for eCryptFS + fixture_name = fixture_name[:70] + "..." + fixture_name[-70:] + return base_dump_dir / fixture_path.stem / fixture_name.replace("/", "-") @pytest.fixture @@ -122,3 +104,26 @@ def fixture_path(test_case: TestCaseIndexFile | TestCaseStream, fixtures_source: def fixture_name(test_case: TestCaseIndexFile | TestCaseStream): """Name of the current fixture.""" return test_case.id + + +def pytest_generate_tests(metafunc): + """Parametrize test cases for every fixture consumer.""" + metafunc.parametrize( + "fixture_consumer", + ( + pytest.param(fixture_consumer, id=str(fixture_consumer.__class__.__name__)) + for fixture_consumer in metafunc.config.fixture_consumers + ), + ) + + +def pytest_collection_modifyitems(items: List): + """ + Modify collected item names to remove the test cases that cannot be consumed by the + given fixture consumer. + """ + for item in items[:]: # use a copy of the list, as we'll be modifying it + fixture_consumer = item.callspec.params["fixture_consumer"] + fixture_format = item.callspec.params["fixture_format"] + if not fixture_consumer.can_consume(fixture_format): + items.remove(item) diff --git a/src/pytest_plugins/consume/direct/test_via_direct.py b/src/pytest_plugins/consume/direct/test_via_direct.py index 1ca8219f470..3d12f97b368 100644 --- a/src/pytest_plugins/consume/direct/test_via_direct.py +++ b/src/pytest_plugins/consume/direct/test_via_direct.py @@ -3,75 +3,29 @@ client interface similar to geth's EVM 'blocktest' command. """ -import re from pathlib import Path -from typing import Any, List, Optional -import pytest - -from ethereum_clis import TransitionTool -from ethereum_test_fixtures import BlockchainFixture, FixtureFormat, StateFixture +from ethereum_test_fixtures import BaseFixture, FixtureConsumer, FixtureFormat from ethereum_test_fixtures.consume import TestCaseIndexFile, TestCaseStream from ..decorator import fixture_format -statetest_results: dict[Path, List[dict[str, Any]]] = {} - -@fixture_format(BlockchainFixture) -def test_blocktest( # noqa: D103 +@fixture_format(*BaseFixture.formats.values()) +def test_fixture( test_case: TestCaseIndexFile | TestCaseStream, - evm: TransitionTool, - evm_run_single_test: bool, + fixture_consumer: FixtureConsumer, fixture_path: Path, fixture_format: FixtureFormat, - test_dump_dir: Optional[Path], + test_dump_dir: Path | None, ): - assert fixture_format == BlockchainFixture - fixture_name = None - if evm_run_single_test: - fixture_name = re.escape(test_case.id) - evm.verify_fixture( - test_case.format, + """ + Generic test function used to call the fixture consumer with a given fixture file path and + a fixture name (for a single test run). + """ + fixture_consumer.consume_fixture( + fixture_format, fixture_path, - fixture_name=fixture_name, + fixture_name=test_case.id, debug_output_path=test_dump_dir, ) - - -@pytest.fixture(scope="function") -def run_statetest( - test_case: TestCaseIndexFile | TestCaseStream, - evm: TransitionTool, - fixture_path: Path, - test_dump_dir: Optional[Path], -): - """Run statetest on the json fixture file if the test result is not already cached.""" - # TODO: Check if all required results have been tested and delete test result data if so. - # TODO: Can we group the tests appropriately so that this works more efficiently with xdist? - if fixture_path not in statetest_results: - json_result = evm.verify_fixture( - test_case.format, - fixture_path, - fixture_name=None, - debug_output_path=test_dump_dir, - ) - statetest_results[fixture_path] = json_result - - -@pytest.mark.usefixtures("run_statetest") -@fixture_format(StateFixture) -def test_statetest( # noqa: D103 - test_case: TestCaseIndexFile | TestCaseStream, - fixture_format: FixtureFormat, - fixture_path: Path, -): - assert fixture_format == StateFixture - test_result = [ - test_result - for test_result in statetest_results[fixture_path] - if test_result["name"] == test_case.id - ] - assert len(test_result) < 2, f"Multiple test results for {test_case.id}" - assert len(test_result) == 1, f"Test result for {test_case.id} missing" - assert test_result[0]["pass"], f"State test failed: {test_result[0]['error']}" diff --git a/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py b/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py index cef6e68073f..1727e37d8dc 100644 --- a/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py +++ b/src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py @@ -5,7 +5,7 @@ Each `engine_newPayloadVX` is verified against the appropriate VALID/INVALID responses. """ -from ethereum_test_fixtures import BlockchainEngineFixture +from ethereum_test_fixtures import BlockchainEngineFixture, FixtureFormat from ethereum_test_rpc import EngineRPC, EthRPC from ethereum_test_rpc.types import ForkchoiceState, JSONRPCError, PayloadStatusEnum from pytest_plugins.consume.hive_simulators.exceptions import GenesisBlockMismatchExceptionError @@ -20,6 +20,7 @@ def test_blockchain_via_engine( eth_rpc: EthRPC, engine_rpc: EngineRPC, fixture: BlockchainEngineFixture, + fixture_format: FixtureFormat, ): """ 1. Check the client genesis block hash matches `fixture.genesis.block_hash`. diff --git a/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py b/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py index 56b73f74096..d7cdffb95ee 100644 --- a/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py +++ b/src/pytest_plugins/consume/hive_simulators/rlp/test_via_rlp.py @@ -5,7 +5,7 @@ Clients consume the genesis and RLP-encoded blocks from input files upon start-up. """ -from ethereum_test_fixtures import BlockchainFixture +from ethereum_test_fixtures import BlockchainFixture, FixtureFormat from ethereum_test_rpc import EthRPC from pytest_plugins.consume.hive_simulators.exceptions import GenesisBlockMismatchExceptionError @@ -18,6 +18,7 @@ def test_via_rlp( timing_data: TimingData, eth_rpc: EthRPC, fixture: BlockchainFixture, + fixture_format: FixtureFormat, ): """ 1. Check the client genesis block hash matches `fixture.genesis.block_hash`. diff --git a/src/pytest_plugins/execute/execute.py b/src/pytest_plugins/execute/execute.py index 7f075f587ae..cb1b2d28dcf 100644 --- a/src/pytest_plugins/execute/execute.py +++ b/src/pytest_plugins/execute/execute.py @@ -353,13 +353,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item]): - """ - Remove pre-Paris tests parametrized to generate hive type fixtures; these - can't be used in the Hive Pyspec Simulator. - - This can't be handled in this plugins pytest_generate_tests() as the fork - parametrization occurs in the forks plugin. - """ + """Remove transition tests and add the appropriate execute markers to the test.""" for item in items[:]: # use a copy of the list, as we'll be modifying it if isinstance(item, EIPSpecTestItem): continue @@ -379,6 +373,6 @@ def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Item for mark in marker.args: item.add_marker(mark) elif marker.name == "valid_at_transition_to": - item.add_marker(pytest.mark.skip(reason="transition tests not executable")) + items.remove(item) if "yul" in item.fixturenames: # type: ignore item.add_marker(pytest.mark.yul_test) diff --git a/src/pytest_plugins/filler/filler.py b/src/pytest_plugins/filler/filler.py index 7f8585a1028..fe77cad4c3b 100644 --- a/src/pytest_plugins/filler/filler.py +++ b/src/pytest_plugins/filler/filler.py @@ -22,8 +22,9 @@ from cli.gen_index import generate_fixtures_index from config import AppConfig from ethereum_clis import TransitionTool +from ethereum_clis.clis.geth import FixtureConsumerTool from ethereum_test_base_types import Alloc, ReferenceSpec -from ethereum_test_fixtures import BaseFixture, FixtureCollector, TestInfo +from ethereum_test_fixtures import BaseFixture, FixtureCollector, FixtureConsumer, TestInfo from ethereum_test_forks import Fork from ethereum_test_specs import SPEC_TYPES, BaseTest from ethereum_test_tools.utility.versioning import ( @@ -428,10 +429,11 @@ def do_fixture_verification( @pytest.fixture(autouse=True, scope="session") def evm_fixture_verification( + request: pytest.FixtureRequest, do_fixture_verification: bool, evm_bin: Path, verify_fixtures_bin: Path | None, -) -> Generator[TransitionTool | None, None, None]: +) -> Generator[FixtureConsumer | None, None, None]: """ Return configured evm binary for executing statetest and blocktest commands used to verify generated JSON fixtures. @@ -439,17 +441,33 @@ def evm_fixture_verification( if not do_fixture_verification: yield None return + reused_evm_bin = False if not verify_fixtures_bin and evm_bin: verify_fixtures_bin = evm_bin - evm_fixture_verification = TransitionTool.from_binary_path(binary_path=verify_fixtures_bin) - if not evm_fixture_verification.blocktest_subcommand: - pytest.exit( - "Only geth's evm tool is supported to verify fixtures: " - "Either remove --verify-fixtures or set --verify-fixtures-bin to a Geth evm binary.", - returncode=pytest.ExitCode.USAGE_ERROR, + reused_evm_bin = True + if not verify_fixtures_bin: + return + try: + evm_fixture_verification = FixtureConsumerTool.from_binary_path( + binary_path=Path(verify_fixtures_bin), + trace=request.config.getoption("evm_collect_traces"), ) + except Exception: + if reused_evm_bin: + pytest.exit( + "The binary specified in --evm-bin could not be recognized as a known " + "FixtureConsumerTool. Either remove --verify-fixtures or set " + "--verify-fixtures-bin to a known fixture consumer binary.", + returncode=pytest.ExitCode.USAGE_ERROR, + ) + else: + pytest.exit( + "Specified binary in --verify-fixtures-bin could not be recognized as a known " + "FixtureConsumerTool. Please see `GethFixtureConsumer` for an example " + "of how a new fixture consumer can be defined.", + returncode=pytest.ExitCode.USAGE_ERROR, + ) yield evm_fixture_verification - evm_fixture_verification.shutdown() @pytest.fixture(scope="session") @@ -580,7 +598,7 @@ def get_fixture_collection_scope(fixture_name, config): def fixture_collector( request: pytest.FixtureRequest, do_fixture_verification: bool, - evm_fixture_verification: TransitionTool, + evm_fixture_verification: FixtureConsumer, filler_path: Path, base_dump_dir: Path | None, output_dir: Path, diff --git a/whitelist.txt b/whitelist.txt index 33a938a2953..9cfa4d8ee70 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -540,6 +540,8 @@ makereport metafunc modifyitems monkeypatching +neth +nethtest nodeid noop oog @@ -585,6 +587,7 @@ tryfirst trylast usefixtures verifier +verifiers writelines xfail ZeroPaddedHexNumber @@ -750,6 +753,7 @@ callcode return delegatecall eofcreate +eoftest extcall extcalls extdelegatecall