From d2b5b3a67b0ca88b1804da30ed619cc5ba954eca Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 28 Aug 2023 20:02:45 +0900 Subject: [PATCH] add a dumb timeout logic https://github.com/WebAssembly/wasi-testsuite/issues/42 --- .../wasi_test_runner/reporters/console.py | 47 +++++++++++-------- .../wasi_test_runner/reporters/json.py | 8 +++- .../wasi_test_runner/runtime_adapter.py | 1 + test-runner/wasi_test_runner/test_case.py | 8 ++++ test-runner/wasi_test_runner/test_suite.py | 12 +++-- .../wasi_test_runner/test_suite_runner.py | 13 +++-- 6 files changed, 60 insertions(+), 29 deletions(-) diff --git a/test-runner/wasi_test_runner/reporters/console.py b/test-runner/wasi_test_runner/reporters/console.py index 8a849c2e..8c23d42c 100644 --- a/test-runner/wasi_test_runner/reporters/console.py +++ b/test-runner/wasi_test_runner/reporters/console.py @@ -2,7 +2,7 @@ from colorama import Fore, init from . import TestReporter -from ..test_case import TestCase +from ..test_case import TestCase, SkippedResult, TimedoutResult from ..test_suite import TestSuite from ..runtime_adapter import RuntimeVersion @@ -20,18 +20,21 @@ def __init__(self, colored: bool = True) -> None: self._colored = colored def report_test(self, test: TestCase) -> None: - if test.result.failed: - self._print_fail(f"Test {test.name} failed") - for reason in test.result.failures: - self._print_fail(f" [{reason.type}] {reason.message}") - print("STDOUT:") - print(test.result.output.stdout) - print("STDERR:") - print(test.result.output.stderr) - elif test.result.is_executed: - self._print_pass(f"Test {test.name} passed") - else: + if isinstance(test.result, TimedoutResult): + self._print_fail(f"Test {test.name} timed out") + elif isinstance(test.result, SkippedResult): self._print_skip(f"Test {test.name} skipped") + else: + if test.result.failed: + self._print_fail(f"Test {test.name} failed") + for reason in test.result.failures: + self._print_fail(f" [{reason.type}] {reason.message}") + print("STDOUT:") + print(test.result.output.stdout) + print("STDERR:") + print(test.result.output.stderr) + else: + self._print_pass(f"Test {test.name} passed") def report_test_suite(self, test_suite: TestSuite) -> None: self._test_suites.append(test_suite) @@ -41,29 +44,31 @@ def finalize(self, version: RuntimeVersion) -> None: print("===== Test results =====") print(f"Runtime: {version.name} {version.version}") - total_skip = total_pass = total_fail = pass_suite = 0 + total_skip = total_pass = total_fail = total_timedout = pass_suite = 0 for suite in self._test_suites: total_pass += suite.pass_count total_fail += suite.fail_count total_skip += suite.skip_count + total_timedout += suite.timedout_count - if suite.fail_count == 0: + if suite.fail_count == 0 and suite.timedout_count == 0: pass_suite += 1 print(f"Suite: {suite.name}") print(f" Total: {suite.test_count}") - self._print_pass(f" Passed: {suite.pass_count}") - self._print_fail(f" Failed: {suite.fail_count}") - self._print_skip(f" Skipped: {suite.skip_count}") + self._print_pass(f" Passed: {suite.pass_count}") + self._print_fail(f" Failed: {suite.fail_count}") + self._print_skip(f" Skipped: {suite.skip_count}") + self._print_fail(f" Timed out: {suite.timedout_count}") print("") print( - f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0)}" + f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0, 0)}" ) - print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip)}") + print(f"Tests: {self._get_summary(total_fail, total_pass, total_skip, total_timedout)}") - def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str: + def _get_summary(self, fail_count: int, pass_count: int, skip_count: int, timedout_count: int) -> str: items: List[str] = [] if fail_count: @@ -72,6 +77,8 @@ def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str items.append(f"{self._pass_color}{pass_count} passed") if skip_count: items.append(f"{self._skip_color}{skip_count} skipped") + if timedout_count: + items.append(f"{self._fail_color}{timedout_count} timed out") total = fail_count + pass_count + skip_count items.append(f"{self._reset_color}{total} total") diff --git a/test-runner/wasi_test_runner/reporters/json.py b/test-runner/wasi_test_runner/reporters/json.py index 023f5563..3a4e9606 100644 --- a/test-runner/wasi_test_runner/reporters/json.py +++ b/test-runner/wasi_test_runner/reporters/json.py @@ -5,6 +5,7 @@ from . import TestReporter from ..test_suite import TestSuite +from ..test_case import Result, SkippedResult, TimedoutResult from ..runtime_adapter import RuntimeVersion @@ -28,16 +29,19 @@ def finalize(self, version: RuntimeVersion) -> None: "duration_s": suite.duration_s, "failed": suite.fail_count, "skipped": suite.skip_count, + "timedout": suite.timedout_count, "passed": suite.pass_count, "tests": [ { "name": test.name, - "executed": test.result.is_executed, + "executed": isinstance(test.result, Result), + "skipped": isinstance(test.result, SkippedResult), + "timedout": isinstance(test.result, TimedoutResult), "duration_s": test.duration_s, "wasi_functions": test.config.wasi_functions, "failures": [ failure.message for failure in test.result.failures - ], + ] if isinstance(test.result, Result) else [], } for test in suite.test_cases ], diff --git a/test-runner/wasi_test_runner/runtime_adapter.py b/test-runner/wasi_test_runner/runtime_adapter.py index 940c2e2c..dbc50a9a 100644 --- a/test-runner/wasi_test_runner/runtime_adapter.py +++ b/test-runner/wasi_test_runner/runtime_adapter.py @@ -48,6 +48,7 @@ def run_test( text=True, check=False, cwd=Path(test_path).parent, + timeout=3. ) return Output(result.returncode, result.stdout, result.stderr) diff --git a/test-runner/wasi_test_runner/test_case.py b/test-runner/wasi_test_runner/test_case.py index eebb95d3..d6cbae93 100644 --- a/test-runner/wasi_test_runner/test_case.py +++ b/test-runner/wasi_test_runner/test_case.py @@ -24,6 +24,14 @@ def failed(self) -> bool: return len(self.failures) > 0 +class TimedoutResult(NamedTuple): + pass + + +class SkippedResult(NamedTuple): + pass + + T = TypeVar("T", bound="Config") diff --git a/test-runner/wasi_test_runner/test_suite.py b/test-runner/wasi_test_runner/test_suite.py index 4681cd9a..09b1b9d3 100644 --- a/test-runner/wasi_test_runner/test_suite.py +++ b/test-runner/wasi_test_runner/test_suite.py @@ -1,6 +1,6 @@ from typing import NamedTuple, List from datetime import datetime -from .test_case import TestCase +from .test_case import TestCase, Result, SkippedResult, TimedoutResult class TestSuite(NamedTuple): @@ -19,7 +19,7 @@ def pass_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed is False + if isinstance(test.result, Result) and test.result.failed is False ] ) @@ -29,10 +29,14 @@ def fail_count(self) -> int: [ 1 for test in self.test_cases - if test.result.is_executed and test.result.failed + if isinstance(test.result, Result) and test.result.failed ] ) @property def skip_count(self) -> int: - return len([1 for test in self.test_cases if not test.result.is_executed]) + return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)]) + + @property + def timedout_count(self) -> int: + return len([1 for test in self.test_cases if isinstance(test.result, TimedoutResult)]) diff --git a/test-runner/wasi_test_runner/test_suite_runner.py b/test-runner/wasi_test_runner/test_suite_runner.py index c9a26b14..32360dda 100644 --- a/test-runner/wasi_test_runner/test_suite_runner.py +++ b/test-runner/wasi_test_runner/test_suite_runner.py @@ -3,6 +3,7 @@ import os import re import shutil +import subprocess import time from datetime import datetime @@ -12,6 +13,8 @@ from .runtime_adapter import RuntimeAdapter from .test_case import ( Result, + SkippedResult, + TimedoutResult, Config, Output, TestCase, @@ -67,7 +70,7 @@ def _skip_single_test( return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=Result(output=Output(0, "", ""), is_executed=False, failures=[]), + result=SkippedResult(), duration_s=0, ) @@ -77,13 +80,17 @@ def _execute_single_test( ) -> TestCase: config = _read_test_config(test_path) test_start = time.time() - test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) + try: + test_output = runtime.run_test(test_path, config.args, config.env, config.dirs) + result=_validate(validators, config, test_output) + except subprocess.TimeoutExpired: + result=TimedoutResult() elapsed = time.time() - test_start return TestCase( name=os.path.splitext(os.path.basename(test_path))[0], config=config, - result=_validate(validators, config, test_output), + result=result, duration_s=elapsed, )