Thread a control flag for asyncio debug through the CLI and Runner (#342

)

Summary:
Pull Request resolved: #342

Context
==

The existing TestSlide runner always enables the asyncio debug flag, which enables slow callback detection etcetera.

There can be cases where the server running the test is overloaded, and random tests will fail due to Python being unable to execute the async callback fast enough. A flag to control this behaviour can be useful when code is known to have slow async callbacks - the source should still be addressed, but disabling the test-failing behaviour enables the code author to check that everything else is ok.

Implementation
==

* Add CLI support for `--slow-callback-is-not-fatal`
* Add Runner support for an `slow_callback_is_not_fatal` parameter that defaults `False` to keep existing behaviour
* Fix a typo in `dsl_unittest.py`

Reviewed By: deathowl

Differential Revision: D41802046

fbshipit-source-id: 60a5e6db71690b6a421d9bf91940820c8be58f43

tests/dsl_unittest.py

@@ -114,8 +114,8 @@ class TestDSLBase(unittest.TestCase):
     def setUp(self):
         reset()
 
-    def run_example(self, exapmle: Example) -> None:
-        _ExampleRunner(exapmle, QuietFormatter(import_module_names=[__name__])).run()
+    def run_example(self, example: Example) -> None:
+        _ExampleRunner(example, QuietFormatter(import_module_names=[__name__])).run()
 
     def run_all_examples(self):
         for each_context in Context.all_top_level_contexts:

testslide/__init__.py

@@ -302,10 +302,16 @@ class SlowCallback(Exception):
 
 
 class _ExampleRunner:
-    def __init__(self, example: "Example", formatter: "BaseFormatter") -> None:
+    def __init__(
+        self,
+        example: "Example",
+        formatter: "BaseFormatter",
+        slow_callback_is_not_fatal: bool = False,
+    ) -> None:
         self.example = example
         self.formatter = formatter
         self.trim_path_prefix = self.formatter.trim_path_prefix
+        self.slow_callback_is_not_fatal = slow_callback_is_not_fatal
 
     @staticmethod
     async def _fail_if_not_coroutine_function(
@@ -386,7 +392,9 @@ async def async_wrapped() -> None:
             )
 
     @contextlib.contextmanager
-    def _raise_if_asyncio_warnings(self, context_data: _ContextData) -> Iterator[None]:
+    def _raise_if_asyncio_warnings(
+        self, context_data: _ContextData, slow_callback_is_not_fatal: bool = False
+    ) -> Iterator[None]:
         original_showwarning = warnings.showwarning
         caught_failures: List[Union[Exception, str]] = []
 
@@ -425,7 +433,8 @@ def logger_warning(msg: str, *args: Any, **kwargs: Any) -> None:
             else:
                 original_logger_warning(msg, *args, **kwargs)
 
-        asyncio.log.logger.warning = logger_warning  # type: ignore
+        if not slow_callback_is_not_fatal:
+            asyncio.log.logger.warning = logger_warning  # type: ignore
 
         aggregated_exceptions = AggregatedExceptions()
 
@@ -442,7 +451,9 @@ def logger_warning(msg: str, *args: Any, **kwargs: Any) -> None:
 
     def _async_run_all_hooks_and_example(self, context_data: _ContextData) -> None:
         coro = self._real_async_run_all_hooks_and_example(context_data)
-        with self._raise_if_asyncio_warnings(context_data):
+        with self._raise_if_asyncio_warnings(
+            context_data, self.slow_callback_is_not_fatal
+        ):
             asyncio_run(coro)
 
     @staticmethod

testslide/cli.py

@@ -168,6 +168,7 @@ class _Config:
     names_regex_exclude: Optional[Pattern[Any]] = None
     dsl_debug: Optional[bool] = False
     profile_threshold_ms: Optional[int] = None
+    slow_callback_is_not_fatal: bool = False
 
 
 class Cli:
@@ -286,6 +287,11 @@ def _build_parser(self, disable_test_files: bool) -> argparse.ArgumentParser:
                 "more than the given number of ms to import. Experimental."
             ),
         )
+        parser.add_argument(
+            "--slow-callback-is-not-fatal",
+            action="store_true",
+            help="Disable treating slow callback as a test failure",
+        )
         if not disable_test_files:
             parser.add_argument(
                 "test_files",
@@ -379,6 +385,7 @@ def _get_config_from_parsed_args(self, parsed_args: Any) -> _Config:
                 _filename_to_module_name(test_file)
                 for test_file in parsed_args.test_files
             ],
+            slow_callback_is_not_fatal=parsed_args.slow_callback_is_not_fatal,
         )
         return config
 
@@ -428,6 +435,7 @@ def run(self) -> int:
                     names_regex_filter=config.names_regex_filter,
                     names_regex_exclude=config.names_regex_exclude,
                     quiet=config.quiet,
+                    slow_callback_is_not_fatal=not config.slow_callback_is_not_fatal,
                 ).run()
 
 

testslide/runner.py

@@ -774,6 +774,7 @@ def __init__(
         names_regex_filter: Optional[Pattern] = None,
         names_regex_exclude: Optional[Pattern] = None,
         quiet: bool = False,
+        slow_callback_is_not_fatal: bool = False,
     ) -> None:
         self.contexts = contexts
         self.formatter = formatter
@@ -786,6 +787,7 @@ def __init__(
         self.names_regex_filter = names_regex_filter
         self.names_regex_exclude = names_regex_exclude
         self.quiet = quiet
+        self.slow_callback_is_not_fatal = slow_callback_is_not_fatal
 
     def _run_example(self, example: Example) -> None:
         if example.focus and self.fail_if_focused:
@@ -799,7 +801,9 @@ def _run_example(self, example: Example) -> None:
             example_exception = None
             with redirect_stdout(stdout), redirect_stderr(stderr):
                 try:
-                    _ExampleRunner(example, self.formatter).run()
+                    _ExampleRunner(
+                        example, self.formatter, self.slow_callback_is_not_fatal
+                    ).run()
                 except BaseException as ex:
                     example_exception = ex
             if example_exception:
@@ -810,7 +814,9 @@ def _run_example(self, example: Example) -> None:
                         print("stderr:\n{}".format(stderr.getvalue()))
                 raise example_exception
         else:
-            _ExampleRunner(example, self.formatter).run()
+            _ExampleRunner(
+                example, self.formatter, self.slow_callback_is_not_fatal
+            ).run()
 
     def run(self) -> int:
         """