Skip to content

Commit

Permalink
Implement a simple test filter
Browse files Browse the repository at this point in the history
cf. #29
  • Loading branch information
yamt authored and loganek committed Jan 20, 2023
1 parent 81576bb commit 8ad7197
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 5 deletions.
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@ python3 test-runner/wasi_test_runner.py
-r adapters/wasmtime.sh # path to a runtime adapter
```

Optionally you can specify test cases to skip with the `--filter` option.

```bash
python3 test-runner/wasi_test_runner.py \
-t ./tests/assemblyscript/testsuite/ `# path to folders containing .wasm test files` \
./tests/c/testsuite/ \
--filter examples/skip.json \
-r adapters/wasmtime.sh # path to a runtime adapter
```

## Contributing

All contributions are very welcome. Contributors can help with:
Expand Down
5 changes: 5 additions & 0 deletions examples/skip.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"WASI C tests": {
"stat-dev-ino": "d_ino is known broken in this combination of engine and wasi-sdk version."
}
}
1 change: 1 addition & 0 deletions test-runner/.pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,6 @@ disable=
C0114, # Missing module docstring
C0115, # Missing class docstring
C0116, # Missing function or method docstring
R0903, # Too few public methods
[FORMAT]
max-line-length=120
14 changes: 12 additions & 2 deletions test-runner/tests/test_test_suite_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,12 @@ def test_runner_end_to_end() -> None:

reporters = [Mock(), Mock()]

filt = Mock()
filt.should_skip.return_value = (False, None)
filters = [filt]

with patch("glob.glob", return_value=test_paths):
suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters) # type: ignore
suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters) # type: ignore

# Assert manifest was read correctly
assert suite.name == "test-suite"
Expand Down Expand Up @@ -91,9 +95,15 @@ def test_runner_end_to_end() -> None:
for config, output in zip(expected_config, outputs):
validator.assert_any_call(config, output)

# Assert filter calls
for filt in filters:
assert filt.should_skip.call_count == 3
for test_case in expected_test_cases:
filt.should_skip.assert_any_call(suite.name, test_case.name)


@patch("os.path.exists", Mock(return_value=False))
def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [])
suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])

assert suite.name == "my-path"
15 changes: 15 additions & 0 deletions test-runner/wasi_test_runner/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

from .runtime_adapter import RuntimeAdapter
from .harness import run_all_tests
from .filters import TestFilter
from .filters import JSONTestFilter
from .reporters import TestReporter
from .reporters.console import ConsoleTestReporter
from .reporters.json import JSONTestReporter
Expand All @@ -23,6 +25,14 @@ def main() -> int:
nargs="+",
help="Locations of suites (directories with *.wasm test files).",
)
parser.add_argument(
"-f",
"--filter",
required=False,
nargs="+",
default=[],
help="Locations of test filters (JSON files).",
)
parser.add_argument(
"-r", "--runtime-adapter", required=True, help="Path to a runtime adapter."
)
Expand All @@ -45,11 +55,16 @@ def main() -> int:

validators: List[Validator] = [exit_code_validator, stdout_validator]

filters: List[TestFilter] = []
for filt in options.filter:
filters.append(JSONTestFilter(filt))

return run_all_tests(
RuntimeAdapter(options.runtime_adapter),
options.test_suite,
validators,
reporters,
filters,
)


Expand Down
24 changes: 24 additions & 0 deletions test-runner/wasi_test_runner/filters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
from typing import Tuple, Any
from abc import ABC
from abc import abstractmethod

import json


class TestFilter(ABC):
@abstractmethod
def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]:
pass


class JSONTestFilter(TestFilter):
def __init__(self, filename: str) -> None:
with open(filename, encoding="utf-8") as file:
self.filter_dict = json.load(file)

def should_skip(self, test_suite_name: str, test_name: str) -> Tuple[bool, Any]:
test_suite_filter = self.filter_dict.get(test_suite_name)
if test_suite_filter is None:
return False, None
why = test_suite_filter.get(test_name)
return why is not None, why
4 changes: 3 additions & 1 deletion test-runner/wasi_test_runner/harness.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from typing import List

from .filters import TestFilter
from .reporters import TestReporter
from .test_suite_runner import run_tests_from_test_suite
from .runtime_adapter import RuntimeAdapter
Expand All @@ -11,12 +12,13 @@ def run_all_tests(
test_suite_paths: List[str],
validators: List[Validator],
reporters: List[TestReporter],
filters: List[TestFilter],
) -> int:
ret = 0

for test_suite_path in test_suite_paths:
test_suite = run_tests_from_test_suite(
test_suite_path, runtime, validators, reporters
test_suite_path, runtime, validators, reporters, filters,
)
for reporter in reporters:
reporter.report_test_suite(test_suite)
Expand Down
1 change: 1 addition & 0 deletions test-runner/wasi_test_runner/reporters/console.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ def finalize(self, version: RuntimeVersion) -> None:
print(f" Total: {suite.test_count}")
self._print_pass(f" Passed: {suite.pass_count}")
self._print_fail(f" Failed: {suite.fail_count}")
self._print_skip(f" Skipped: {suite.skip_count}")
print("")

print(
Expand Down
29 changes: 27 additions & 2 deletions test-runner/wasi_test_runner/test_suite_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from datetime import datetime
from typing import List, cast

from .filters import TestFilter
from .runtime_adapter import RuntimeAdapter
from .test_case import (
Result,
Expand All @@ -25,28 +26,52 @@ def run_tests_from_test_suite(
runtime: RuntimeAdapter,
validators: List[Validator],
reporters: List[TestReporter],
filters: List[TestFilter],
) -> TestSuite:
test_cases: List[TestCase] = []
test_start = datetime.now()

_cleanup_test_output(test_suite_path)

test_suite_name = _read_manifest(test_suite_path)

for test_path in glob.glob(os.path.join(test_suite_path, "*.wasm")):
test_case = _execute_single_test(runtime, validators, test_path)
test_name = os.path.splitext(os.path.basename(test_path))[0]
for filt in filters:
# for now, just drop the skip reason string. it might be
# useful to make reporters report it.
skip, _ = filt.should_skip(test_suite_name, test_name)
if skip:
test_case = _skip_single_test(runtime, validators, test_path)
break
else:
test_case = _execute_single_test(runtime, validators, test_path)
test_cases.append(test_case)
for reporter in reporters:
reporter.report_test(test_case)

elapsed = (datetime.now() - test_start).total_seconds()

return TestSuite(
name=_read_manifest(test_suite_path),
name=test_suite_name,
time=test_start,
duration_s=elapsed,
test_cases=test_cases,
)


def _skip_single_test(
_runtime: RuntimeAdapter, _validators: List[Validator], test_path: str
) -> TestCase:
config = _read_test_config(test_path)
return TestCase(
name=os.path.splitext(os.path.basename(test_path))[0],
config=config,
result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
duration_s=0,
)


def _execute_single_test(
runtime: RuntimeAdapter, validators: List[Validator], test_path: str
) -> TestCase:
Expand Down

0 comments on commit 8ad7197

Please sign in to comment.