Skip to content

Commit

Permalink
Thread a control flag for asyncio debug through the CLI and Runner (#342
Browse files Browse the repository at this point in the history
)

Summary:
Pull Request resolved: #342

Context
==

The existing TestSlide runner always enables the asyncio debug flag, which enables slow callback detection etcetera.

There can be cases where the server running the test is overloaded, and random tests will fail due to Python being unable to execute the async callback fast enough. A flag to control this behaviour can be useful when code is known to have slow async callbacks - the source should still be addressed, but disabling the test-failing behaviour enables the code author to check that everything else is ok.

Implementation
==

* Add CLI support for `--slow-callback-is-not-fatal`
* Add Runner support for an `slow_callback_is_not_fatal` parameter that defaults `False` to keep existing behaviour
* Fix a typo in `dsl_unittest.py`

Reviewed By: deathowl

Differential Revision: D41802046

fbshipit-source-id: 60a5e6db71690b6a421d9bf91940820c8be58f43
  • Loading branch information
bajanduncan authored and facebook-github-bot committed Dec 16, 2022
1 parent 61b016b commit eb0e389
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 8 deletions.
4 changes: 2 additions & 2 deletions tests/dsl_unittest.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ class TestDSLBase(unittest.TestCase):
def setUp(self):
reset()

def run_example(self, exapmle: Example) -> None:
_ExampleRunner(exapmle, QuietFormatter(import_module_names=[__name__])).run()
def run_example(self, example: Example) -> None:
_ExampleRunner(example, QuietFormatter(import_module_names=[__name__])).run()

def run_all_examples(self):
for each_context in Context.all_top_level_contexts:
Expand Down
19 changes: 15 additions & 4 deletions testslide/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,10 +302,16 @@ class SlowCallback(Exception):


class _ExampleRunner:
def __init__(self, example: "Example", formatter: "BaseFormatter") -> None:
def __init__(
self,
example: "Example",
formatter: "BaseFormatter",
slow_callback_is_not_fatal: bool = False,
) -> None:
self.example = example
self.formatter = formatter
self.trim_path_prefix = self.formatter.trim_path_prefix
self.slow_callback_is_not_fatal = slow_callback_is_not_fatal

@staticmethod
async def _fail_if_not_coroutine_function(
Expand Down Expand Up @@ -386,7 +392,9 @@ async def async_wrapped() -> None:
)

@contextlib.contextmanager
def _raise_if_asyncio_warnings(self, context_data: _ContextData) -> Iterator[None]:
def _raise_if_asyncio_warnings(
self, context_data: _ContextData, slow_callback_is_not_fatal: bool = False
) -> Iterator[None]:
original_showwarning = warnings.showwarning
caught_failures: List[Union[Exception, str]] = []

Expand Down Expand Up @@ -425,7 +433,8 @@ def logger_warning(msg: str, *args: Any, **kwargs: Any) -> None:
else:
original_logger_warning(msg, *args, **kwargs)

asyncio.log.logger.warning = logger_warning # type: ignore
if not slow_callback_is_not_fatal:
asyncio.log.logger.warning = logger_warning # type: ignore

aggregated_exceptions = AggregatedExceptions()

Expand All @@ -442,7 +451,9 @@ def logger_warning(msg: str, *args: Any, **kwargs: Any) -> None:

def _async_run_all_hooks_and_example(self, context_data: _ContextData) -> None:
coro = self._real_async_run_all_hooks_and_example(context_data)
with self._raise_if_asyncio_warnings(context_data):
with self._raise_if_asyncio_warnings(
context_data, self.slow_callback_is_not_fatal
):
asyncio_run(coro)

@staticmethod
Expand Down
8 changes: 8 additions & 0 deletions testslide/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ class _Config:
names_regex_exclude: Optional[Pattern[Any]] = None
dsl_debug: Optional[bool] = False
profile_threshold_ms: Optional[int] = None
slow_callback_is_not_fatal: bool = False


class Cli:
Expand Down Expand Up @@ -286,6 +287,11 @@ def _build_parser(self, disable_test_files: bool) -> argparse.ArgumentParser:
"more than the given number of ms to import. Experimental."
),
)
parser.add_argument(
"--slow-callback-is-not-fatal",
action="store_true",
help="Disable treating slow callback as a test failure",
)
if not disable_test_files:
parser.add_argument(
"test_files",
Expand Down Expand Up @@ -379,6 +385,7 @@ def _get_config_from_parsed_args(self, parsed_args: Any) -> _Config:
_filename_to_module_name(test_file)
for test_file in parsed_args.test_files
],
slow_callback_is_not_fatal=parsed_args.slow_callback_is_not_fatal,
)
return config

Expand Down Expand Up @@ -428,6 +435,7 @@ def run(self) -> int:
names_regex_filter=config.names_regex_filter,
names_regex_exclude=config.names_regex_exclude,
quiet=config.quiet,
slow_callback_is_not_fatal=not config.slow_callback_is_not_fatal,
).run()


Expand Down
10 changes: 8 additions & 2 deletions testslide/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,7 @@ def __init__(
names_regex_filter: Optional[Pattern] = None,
names_regex_exclude: Optional[Pattern] = None,
quiet: bool = False,
slow_callback_is_not_fatal: bool = False,
) -> None:
self.contexts = contexts
self.formatter = formatter
Expand All @@ -786,6 +787,7 @@ def __init__(
self.names_regex_filter = names_regex_filter
self.names_regex_exclude = names_regex_exclude
self.quiet = quiet
self.slow_callback_is_not_fatal = slow_callback_is_not_fatal

def _run_example(self, example: Example) -> None:
if example.focus and self.fail_if_focused:
Expand All @@ -799,7 +801,9 @@ def _run_example(self, example: Example) -> None:
example_exception = None
with redirect_stdout(stdout), redirect_stderr(stderr):
try:
_ExampleRunner(example, self.formatter).run()
_ExampleRunner(
example, self.formatter, self.slow_callback_is_not_fatal
).run()
except BaseException as ex:
example_exception = ex
if example_exception:
Expand All @@ -810,7 +814,9 @@ def _run_example(self, example: Example) -> None:
print("stderr:\n{}".format(stderr.getvalue()))
raise example_exception
else:
_ExampleRunner(example, self.formatter).run()
_ExampleRunner(
example, self.formatter, self.slow_callback_is_not_fatal
).run()

def run(self) -> int:
"""
Expand Down

0 comments on commit eb0e389

Please sign in to comment.