diff --git a/README.md b/README.md index b94bf3a6..d04dde8d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py -r adapters/wasmtime.sh # path to a runtime adapter ``` +Optianally you can specify test cases to skip with the `--filter` option. + +```bash +python3 test-runner/wasi_test_runner.py \ + -t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \ + ./tests/c/testsuite/ \ + --filter examples/skip.json + -r adapters/wasmtime.sh # path to a runtime adapter +``` + ## Contributing All contributions are very welcome. Contributors can help with: diff --git a/examples/skip.json b/examples/skip.json new file mode 100644 index 00000000..daa4827b --- /dev/null +++ b/examples/skip.json @@ -0,0 +1,5 @@ +{ + "WASI C tests": { + "stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version." + } +} diff --git a/test-runner/tests/test_test_suite_runner.py b/test-runner/tests/test_test_suite_runner.py index 75d75c51..7cac2fc7 100644 --- a/test-runner/tests/test_test_suite_runner.py +++ b/test-runner/tests/test_test_suite_runner.py @@ -94,6 +94,6 @@ def test_runner_end_to_end() -> None: @patch("os.path.exists", Mock(return_value=False)) def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None: - suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], []) + suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], []) assert suite.name == "my-path" diff --git a/test-runner/wasi_test_runner/__main__.py b/test-runner/wasi_test_runner/__main__.py index 1eef0e1f..6d20fc50 100644 --- a/test-runner/wasi_test_runner/__main__.py +++ b/test-runner/wasi_test_runner/__main__.py @@ -5,6 +5,8 @@ from .runtime_adapter import RuntimeAdapter from .harness import run_all_tests +from .filters import TestFilter +from .filters import JSONTestFilter from .reporters import TestReporter from .reporters.console import ConsoleTestReporter from .reporters.json import JSONTestReporter @@ -23,6 +25,14 @@ def main() -> int: nargs="+", help="Locations of suites (directories with *.wasm test files).", ) + parser.add_argument( + "-f", + "--filter", + required=False, + nargs="+", + default=[], + help="Locations of test filters (JSON files).", + ) parser.add_argument( "-r", "--runtime-adapter", required=True, help="Path to a runtime adapter." ) @@ -45,11 +55,16 @@ def main() -> int: validators: List[Validator] = [exit_code_validator, stdout_validator] + filters: List[TestFilter] = [] + for filter in options.filter: + filters.append(JSONTestFilter(filter)) + return run_all_tests( RuntimeAdapter(options.runtime_adapter), options.test_suite, validators, reporters, + filters, ) diff --git a/test-runner/wasi_test_runner/filters.py b/test-runner/wasi_test_runner/filters.py new file mode 100644 index 00000000..87310360 --- /dev/null +++ b/test-runner/wasi_test_runner/filters.py @@ -0,0 +1,24 @@ +from typing import Tuple, Any +from abc import ABC +from abc import abstractmethod + +import json + + +class TestFilter(ABC): + @abstractmethod + def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]: + pass + + +class JSONTestFilter(TestFilter): + def __init__(self, filename: str) -> None: + with open(filename) as fp: + self.filter_dict = json.load(fp) + + def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]: + d = self.filter_dict.get(test_suite_name) + if d is None: + return False, None + why = d.get(test_name) + return why is not None, why diff --git a/test-runner/wasi_test_runner/harness.py b/test-runner/wasi_test_runner/harness.py index d440b30a..027b2bb5 100644 --- a/test-runner/wasi_test_runner/harness.py +++ b/test-runner/wasi_test_runner/harness.py @@ -1,5 +1,6 @@ from typing import List +from .filters import TestFilter from .reporters import TestReporter from .test_suite_runner import run_tests_from_test_suite from .runtime_adapter import RuntimeAdapter @@ -11,12 +12,13 @@ def run_all_tests( test_suite_paths: List[str], validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> int: ret = 0 for test_suite_path in test_suite_paths: test_suite = run_tests_from_test_suite( - test_suite_path, runtime, validators, reporters + test_suite_path, runtime, validators, reporters, filters, ) for reporter in reporters: reporter.report_test_suite(test_suite) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 9bbae086..2eb6ed96 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None: print(f" Total: {suite.test_count}") self._print_pass(f" Passed: {suite.pass_count}") self._print_fail(f" Failed: {suite.fail_count}") + print(f" Skipped: {suite.skip_count}") print("") print( diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index 47a83c9e..4a549454 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -8,6 +8,7 @@ from datetime import datetime from typing import List, cast +from .filters import TestFilter from .runtime_adapter import RuntimeAdapter from .test_case import ( Result, @@ -25,14 +26,26 @@ def run_tests_from_test_suite( runtime: RuntimeAdapter, validators: List[Validator], reporters: List[TestReporter], + filters: List[TestFilter], ) -> TestSuite: test_cases: List[TestCase] = [] test_start = datetime.now() _cleanup_test_output(test_suite_path) + test_suite_name = _read_manifest(test_suite_path) + for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")): - test_case = _execute_single_test(runtime, validators, test_path) + test_name = os.path.splitext(os.path.basename(test_path))[0] + for filter in filters: + # for now, just drop the skip reason string. it might be + # useful to make reporters report it. + skip, _ = filter.should_skip(test_suite_name, test_name) + if skip: + test_case = _skip_single_test(runtime, validators, test_path) + break + else: + test_case = _execute_single_test(runtime, validators, test_path) test_cases.append(test_case) for reporter in reporters: reporter.report_test(test_case) @@ -40,13 +53,25 @@ def run_tests_from_test_suite( elapsed = (datetime.now() - test_start).total_seconds() return TestSuite( - name=_read_manifest(test_suite_path), + name=test_suite_name, time=test_start, duration_s=elapsed, test_cases=test_cases, ) +def _skip_single_test( + _: RuntimeAdapter, validators: List[Validator], test_path: str +) -> TestCase: + config = _read_test_config(test_path) + return TestCase( + name=os.path.splitext(os.path.basename(test_path))[0], + config=config, + result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + duration_s=0, + ) + + def _execute_single_test( runtime: RuntimeAdapter, validators: List[Validator], test_path: str ) -> TestCase: