add a dumb timeout logic

WebAssembly#42

test-runner/wasi_test_runner/reporters/console.py

@@ -2,7 +2,7 @@
 from colorama import Fore, init
 
 from . import TestReporter
-from ..test_case import TestCase
+from ..test_case import TestCase, SkippedResult, TimedoutResult
 from ..test_suite import TestSuite
 from ..runtime_adapter import RuntimeVersion
 
@@ -20,18 +20,21 @@ def __init__(self, colored: bool = True) -> None:
         self._colored = colored
 
     def report_test(self, test: TestCase) -> None:
-        if test.result.failed:
-            self._print_fail(f"Test {test.name} failed")
-            for reason in test.result.failures:
-                self._print_fail(f"  [{reason.type}] {reason.message}")
-            print("STDOUT:")
-            print(test.result.output.stdout)
-            print("STDERR:")
-            print(test.result.output.stderr)
-        elif test.result.is_executed:
-            self._print_pass(f"Test {test.name} passed")
-        else:
+        if isinstance(test.result, TimedoutResult):
+            self._print_fail(f"Test {test.name} timed out")
+        elif isinstance(test.result, SkippedResult):
             self._print_skip(f"Test {test.name} skipped")
+        else:
+            if test.result.failed:
+                self._print_fail(f"Test {test.name} failed")
+                for reason in test.result.failures:
+                    self._print_fail(f"  [{reason.type}] {reason.message}")
+                print("STDOUT:")
+                print(test.result.output.stdout)
+                print("STDERR:")
+                print(test.result.output.stderr)
+            else:
+                self._print_pass(f"Test {test.name} passed")
 
     def report_test_suite(self, test_suite: TestSuite) -> None:
         self._test_suites.append(test_suite)
@@ -41,29 +44,31 @@ def finalize(self, version: RuntimeVersion) -> None:
         print("===== Test results =====")
         print(f"Runtime: {version.name} {version.version}")
 
-        total_skip = total_pass = total_fail = pass_suite = 0
+        total_skip = total_pass = total_fail = total_timedout = pass_suite = 0
 
         for suite in self._test_suites:
             total_pass += suite.pass_count
             total_fail += suite.fail_count
             total_skip += suite.skip_count
+            total_timedout += suite.timedout_count
 
-            if suite.fail_count == 0:
+            if suite.fail_count == 0 and suite.timedout_count == 0:
                 pass_suite += 1
 
             print(f"Suite: {suite.name}")
             print(f"  Total: {suite.test_count}")
-            self._print_pass(f"  Passed:  {suite.pass_count}")
-            self._print_fail(f"  Failed:  {suite.fail_count}")
-            self._print_skip(f"  Skipped: {suite.skip_count}")
+            self._print_pass(f"  Passed:    {suite.pass_count}")
+            self._print_fail(f"  Failed:    {suite.fail_count}")
+            self._print_skip(f"  Skipped:   {suite.skip_count}")
+            self._print_fail(f"  Timed out: {suite.timedout_count}")
             print("")
 
         print(
-            f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0)}"
+            f"Test suites: {self._get_summary(len(self._test_suites) - pass_suite, pass_suite, 0, 0)}"
         )
-        print(f"Tests:       {self._get_summary(total_fail, total_pass, total_skip)}")
+        print(f"Tests:       {self._get_summary(total_fail, total_pass, total_skip, total_timedout)}")
 
-    def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str:
+    def _get_summary(self, fail_count: int, pass_count: int, skip_count: int, timedout_count: int) -> str:
         items: List[str] = []
 
         if fail_count:
@@ -72,6 +77,8 @@ def _get_summary(self, fail_count: int, pass_count: int, skip_count: int) -> str
             items.append(f"{self._pass_color}{pass_count} passed")
         if skip_count:
             items.append(f"{self._skip_color}{skip_count} skipped")
+        if timedout_count:
+            items.append(f"{self._fail_color}{timedout_count} timed out")
 
         total = fail_count + pass_count + skip_count
         items.append(f"{self._reset_color}{total} total")

test-runner/wasi_test_runner/reporters/json.py

@@ -5,6 +5,7 @@
 
 from . import TestReporter
 from ..test_suite import TestSuite
+from ..test_case import Result, SkippedResult, TimedoutResult
 from ..runtime_adapter import RuntimeVersion
 
 
@@ -28,16 +29,19 @@ def finalize(self, version: RuntimeVersion) -> None:
                     "duration_s": suite.duration_s,
                     "failed": suite.fail_count,
                     "skipped": suite.skip_count,
+                    "timedout": suite.timedout_count,
                     "passed": suite.pass_count,
                     "tests": [
                         {
                             "name": test.name,
-                            "executed": test.result.is_executed,
+                            "executed": isinstance(test.result, Result),
+                            "skipped": isinstance(test.result, SkippedResult),
+                            "timedout": isinstance(test.result, TimedoutResult),
                             "duration_s": test.duration_s,
                             "wasi_functions": test.config.wasi_functions,
                             "failures": [
                                 failure.message for failure in test.result.failures
-                            ],
+                            ] if isinstance(test.result, Result) else [],
                         }
                         for test in suite.test_cases
                     ],

test-runner/wasi_test_runner/runtime_adapter.py

@@ -48,6 +48,7 @@ def run_test(
             text=True,
             check=False,
             cwd=Path(test_path).parent,
+            timeout=3.
         )
         return Output(result.returncode, result.stdout, result.stderr)
 

test-runner/wasi_test_runner/test_case.py

@@ -24,6 +24,14 @@ def failed(self) -> bool:
         return len(self.failures) > 0
 
 
+class TimedoutResult(NamedTuple):
+    pass
+
+
+class SkippedResult(NamedTuple):
+    pass
+
+
 T = TypeVar("T", bound="Config")
 
 

test-runner/wasi_test_runner/test_suite.py

@@ -1,6 +1,6 @@
 from typing import NamedTuple, List
 from datetime import datetime
-from .test_case import TestCase
+from .test_case import TestCase, Result, SkippedResult, TimedoutResult
 
 
 class TestSuite(NamedTuple):
@@ -19,7 +19,7 @@ def pass_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed is False
+                if isinstance(test.result, Result) and test.result.failed is False
             ]
         )
 
@@ -29,10 +29,14 @@ def fail_count(self) -> int:
             [
                 1
                 for test in self.test_cases
-                if test.result.is_executed and test.result.failed
+                if isinstance(test.result, Result) and test.result.failed
             ]
         )
 
     @property
     def skip_count(self) -> int:
-        return len([1 for test in self.test_cases if not test.result.is_executed])
+        return len([1 for test in self.test_cases if isinstance(test.result, SkippedResult)])
+
+    @property
+    def timedout_count(self) -> int:
+        return len([1 for test in self.test_cases if isinstance(test.result, TimedoutResult)])

test-runner/wasi_test_runner/test_suite_runner.py

@@ -3,6 +3,7 @@
 import os
 import re
 import shutil
+import subprocess
 import time
 
 from datetime import datetime
@@ -12,6 +13,8 @@
 from .runtime_adapter import RuntimeAdapter
 from .test_case import (
     Result,
+    SkippedResult,
+    TimedoutResult,
     Config,
     Output,
     TestCase,
@@ -67,7 +70,7 @@ def _skip_single_test(
     return TestCase(
         name=os.path.splitext(os.path.basename(test_path))[0],
         config=config,
-        result=Result(output=Output(0, "", ""), is_executed=False, failures=[]),
+        result=SkippedResult(),
         duration_s=0,
     )
 
@@ -77,13 +80,17 @@ def _execute_single_test(
 ) -> TestCase:
     config = _read_test_config(test_path)
     test_start = time.time()
-    test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
+    try:
+        test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
+        result=_validate(validators, config, test_output)
+    except subprocess.TimeoutExpired:
+        result=TimedoutResult()
     elapsed = time.time() - test_start
 
     return TestCase(
         name=os.path.splitext(os.path.basename(test_path))[0],
         config=config,
-        result=_validate(validators, config, test_output),
+        result=result,
         duration_s=elapsed,
     )