From cca2cf3990c592bb7981fd79966449ab83b77670 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 7 Dec 2018 00:11:40 +0100 Subject: [PATCH 01/15] Make `doCleanups` work for the DeferrableTestCase --- tests/test_ensure_do_cleanups.py | 15 +++++++++++++++ unittesting/core/st3/case.py | 17 ++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests/test_ensure_do_cleanups.py diff --git a/tests/test_ensure_do_cleanups.py b/tests/test_ensure_do_cleanups.py new file mode 100644 index 00000000..3d60c62f --- /dev/null +++ b/tests/test_ensure_do_cleanups.py @@ -0,0 +1,15 @@ +from unittesting import DeferrableTestCase + + +class TestDoCleanups(DeferrableTestCase): + + def test_ensure_do_cleanups_works(self): + messages = [] + + def work(message): + messages.append(message) + + self.addCleanup(work, 1) + yield from self.doCleanups() + + self.assertEqual(messages, [1]) diff --git a/unittesting/core/st3/case.py b/unittesting/core/st3/case.py index 9d52e8b5..c3164cc1 100644 --- a/unittesting/core/st3/case.py +++ b/unittesting/core/st3/case.py @@ -74,7 +74,7 @@ def run(self, result=None): if isiterable(deferred): yield from deferred - self.doCleanups() + yield from self.doCleanups() if outcome.success: result.addSuccess(self) else: @@ -110,3 +110,18 @@ def run(self, result=None): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() + + def doCleanups(self): + """Execute all cleanup functions. + + Normally called for you after tearDown. + """ + outcome = self._outcomeForDoCleanups or _Outcome() + while self._cleanups: + function, args, kwargs = self._cleanups.pop() + part = lambda: function(*args, **kwargs) # noqa: E731 + yield from self._executeTestPart(part, outcome) + + # return this for backwards compatibility + # even though we no longer us it internally + return outcome.success From 18b20a4b76d66d2faeecac04510d2f56beaa6ca7 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 7 Dec 2018 00:21:39 +0100 Subject: [PATCH 02/15] `_executeTestPart` is a generator and always iterable --- unittesting/core/st3/case.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/unittesting/core/st3/case.py b/unittesting/core/st3/case.py index c3164cc1..29e4123c 100644 --- a/unittesting/core/st3/case.py +++ b/unittesting/core/st3/case.py @@ -63,16 +63,10 @@ def run(self, result=None): outcome = _Outcome() self._outcomeForDoCleanups = outcome - deferred = self._executeTestPart(self.setUp, outcome) - if isiterable(deferred): - yield from deferred + yield from self._executeTestPart(self.setUp, outcome) if outcome.success: - deferred = self._executeTestPart(testMethod, outcome, isTest=True) - if isiterable(deferred): - yield from deferred - deferred = self._executeTestPart(self.tearDown, outcome) - if isiterable(deferred): - yield from deferred + yield from self._executeTestPart(testMethod, outcome, isTest=True) + yield from self._executeTestPart(self.tearDown, outcome) yield from self.doCleanups() if outcome.success: From 79535f03fa0a2fb9eb3dde7a548a952f3854880a Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 7 Dec 2018 01:05:03 +0100 Subject: [PATCH 03/15] UX: Remember last current file and use it If a user switches from test file to implementation file, it is useful that 'Test Current File' remembers and executes the actual last test file run. Changes: - We actually match the filename against `settings['pattern']`. Not all files with a '.py' suffix are test files. - We store the last used test file in the `window.settings()` which is cheap to implement and probably the right thing. --- unittesting/mixin.py | 7 ------- unittesting/test_current.py | 19 ++++++++++++++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/unittesting/mixin.py b/unittesting/mixin.py index f4b490c3..2f8b1888 100644 --- a/unittesting/mixin.py +++ b/unittesting/mixin.py @@ -62,13 +62,6 @@ def current_package_name(self): return None - @property - def current_test_file(self): - current_file = sublime.active_window().active_view().file_name() - if current_file and current_file.endswith(".py"): - current_file = os.path.basename(current_file) - return current_file - def input_parser(self, package): m = re.match(r'([^:]+):(.+)', package) if m: diff --git a/unittesting/test_current.py b/unittesting/test_current.py index 16a321ea..17303272 100644 --- a/unittesting/test_current.py +++ b/unittesting/test_current.py @@ -1,3 +1,5 @@ +from fnmatch import fnmatch +import os import sublime import sys from .test_package import UnitTestingCommand @@ -43,9 +45,20 @@ def run(self, **kwargs): sublime.message_dialog("Cannot determine package name.") return - test_file = self.current_test_file - if not test_file: - test_file = "" + window = sublime.active_window() + if not window: + return + + view = window.active_view() + + settings = self.load_unittesting_settings(project_name, kwargs) + current_file = (view and view.file_name()) or '' + file_name = os.path.basename(current_file) + if file_name and fnmatch(file_name, settings['pattern']): + test_file = file_name + window.settings().set('last_test_file', test_file) + else: + test_file = window.settings().get('last_test_file') or current_file sublime.set_timeout_async( lambda: super(UnitTestingCurrentFileCommand, self).run( From d3204274fa0bf2cd507d5e72b9de5c434c46ab66 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 7 Dec 2018 20:45:28 +0100 Subject: [PATCH 04/15] Namespace the 'last_test_file' setting key --- .flake8 | 3 ++- unittesting/test_current.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.flake8 b/.flake8 index 16851d91..9efa3bb1 100644 --- a/.flake8 +++ b/.flake8 @@ -7,5 +7,6 @@ max-line-length = 120 # D103 Missing docstring in public function # D104 Missing docstring in public package # D107 Missing docstring in __init__ +# W503 line break before binary operator # W606 'async' and 'await' are reserved keywords starting with Python 3.7 -ignore = D100, D101, D102, D103, D104, D107, W606 +ignore = D100, D101, D102, D103, D104, D107, W503, W606 diff --git a/unittesting/test_current.py b/unittesting/test_current.py index 17303272..6a1e01b9 100644 --- a/unittesting/test_current.py +++ b/unittesting/test_current.py @@ -56,9 +56,12 @@ def run(self, **kwargs): file_name = os.path.basename(current_file) if file_name and fnmatch(file_name, settings['pattern']): test_file = file_name - window.settings().set('last_test_file', test_file) + window.settings().set('UnitTesting.last_test_file', test_file) else: - test_file = window.settings().get('last_test_file') or current_file + test_file = ( + window.settings().get('UnitTesting.last_test_file') + or current_file + ) sublime.set_timeout_async( lambda: super(UnitTestingCurrentFileCommand, self).run( From cbbf178a59561e828214ff88e0c73ec0716fa926 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 7 Dec 2018 23:47:40 +0100 Subject: [PATCH 05/15] Test implicit `doCleanups` on tearDown --- tests/test_ensure_do_cleanups.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_ensure_do_cleanups.py b/tests/test_ensure_do_cleanups.py index 3d60c62f..6a827bd0 100644 --- a/tests/test_ensure_do_cleanups.py +++ b/tests/test_ensure_do_cleanups.py @@ -1,9 +1,9 @@ from unittesting import DeferrableTestCase -class TestDoCleanups(DeferrableTestCase): +class TestExplicitDoCleanups(DeferrableTestCase): - def test_ensure_do_cleanups_works(self): + def test_manually_calling_do_cleanups_works(self): messages = [] def work(message): @@ -13,3 +13,14 @@ def work(message): yield from self.doCleanups() self.assertEqual(messages, [1]) + + +cleanup_called = [] + + +class TestImplicitDoCleanupsOnTeardown(DeferrableTestCase): + def test_a_prepare(self): + self.addCleanup(lambda: cleanup_called.append(1)) + + def test_b_assert(self): + self.assertEqual(cleanup_called, [1]) From 4d1d43e94e94529b09ae64d6561fa92db8794ffe Mon Sep 17 00:00:00 2001 From: herr kaste Date: Fri, 30 Nov 2018 11:32:16 +0100 Subject: [PATCH 06/15] Throw and send the condition outcome into the test --- tests/test_yield_condition.py | 17 +++++++++++++++++ unittesting/core/st3/runner.py | 18 ++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 tests/test_yield_condition.py diff --git a/tests/test_yield_condition.py b/tests/test_yield_condition.py new file mode 100644 index 00000000..1614aeba --- /dev/null +++ b/tests/test_yield_condition.py @@ -0,0 +1,17 @@ +from unittesting import DeferrableTestCase + + +class TestYieldConditionsHandlingInDeferredTestCase(DeferrableTestCase): + def test_reraises_errors_raised_in_conditions(self): + try: + yield lambda: 1 / 0 + self.fail('Did not reraise the exception from the condition') + except ZeroDivisionError: + pass + except Exception: + self.fail('Did not throw the original exception') + + def test_returns_condition_value(self): + rv = yield lambda: 'Hans Peter' + + self.assertEqual(rv, 'Hans Peter') diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index 89623f86..ca62fc0e 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -45,9 +45,13 @@ def _start_testing(): except Exception as e: _handle_error(e) - def _continue_testing(deferred): + def _continue_testing(deferred, send_value=None, throw_value=None): try: - condition = next(deferred) + if throw_value: + condition = deferred.throw(throw_value) + else: + condition = deferred.send(send_value) + if callable(condition): defer(100, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ @@ -70,8 +74,14 @@ def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=N if start_time is None: start_time = time.time() - if condition(): - defer(10, _continue_testing, deferred) + try: + send_value = condition() + except Exception as e: + defer(10, _continue_testing, deferred, throw_value=e) + return + + if send_value: + defer(10, _continue_testing, deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: self.stream.writeln("Condition timeout, continue anyway.") defer(10, _continue_testing, deferred) From 87d8df484b560f19753af86c2abc73362957be70 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Wed, 5 Dec 2018 22:36:17 +0100 Subject: [PATCH 07/15] Add `yield AWAIT_WORKER` to await the worker thread # Conflicts: # unittesting/core/st3/runner.py --- tests/test_await_worker.py | 47 ++++++++++++++++++++++++++++++++++ unittesting/__init__.py | 5 ++-- unittesting/core/__init__.py | 10 ++++++-- unittesting/core/st3/runner.py | 12 +++++++++ 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 tests/test_await_worker.py diff --git a/tests/test_await_worker.py b/tests/test_await_worker.py new file mode 100644 index 00000000..866d34da --- /dev/null +++ b/tests/test_await_worker.py @@ -0,0 +1,47 @@ +from functools import partial +import time + +import sublime + +from unittesting import DeferrableTestCase, AWAIT_WORKER + + +def run_in_worker(fn, *args, **kwargs): + sublime.set_timeout_async(partial(fn, *args, **kwargs)) + + +class TestAwaitingWorkerInDeferredTestCase(DeferrableTestCase): + + def test_ensure_plain_yield_is_faster_than_the_worker_thread(self): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + run_in_worker(work, 1, 5) + + yield + + self.assertEqual(messages, []) + + def test_await_worker(self): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + run_in_worker(work, 1, 0.5) + + yield AWAIT_WORKER + + self.assertEqual(messages, [1]) diff --git a/unittesting/__init__.py b/unittesting/__init__.py index 7648f383..3c2108d6 100644 --- a/unittesting/__init__.py +++ b/unittesting/__init__.py @@ -1,4 +1,4 @@ -from .core import DeferrableTestCase +from .core import DeferrableTestCase, AWAIT_WORKER from .scheduler import UnitTestingRunSchedulerCommand from .scheduler import run_scheduler from .test_package import UnitTestingCommand @@ -22,5 +22,6 @@ "UnitTestingCurrentPackageCoverageCommand", "UnitTestingSyntaxCommand", "UnitTestingSyntaxCompatibilityCommand", - "UnitTestingColorSchemeCommand" + "UnitTestingColorSchemeCommand", + "AWAIT_WORKER" ] diff --git a/unittesting/core/__init__.py b/unittesting/core/__init__.py index 73c6b6a0..3a869e82 100644 --- a/unittesting/core/__init__.py +++ b/unittesting/core/__init__.py @@ -1,6 +1,12 @@ -from .st3.runner import DeferringTextTestRunner +from .st3.runner import DeferringTextTestRunner, AWAIT_WORKER from .st3.case import DeferrableTestCase from .st3.suite import DeferrableTestSuite from .loader import UnitTestingLoader as TestLoader -__all__ = ["TestLoader", "DeferringTextTestRunner", "DeferrableTestCase", "DeferrableTestSuite"] +__all__ = [ + "TestLoader", + "DeferringTextTestRunner", + "DeferrableTestCase", + "DeferrableTestSuite", + "AWAIT_WORKER" +] diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index ca62fc0e..dfa3f7f1 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -1,3 +1,4 @@ +from functools import partial import time from unittest.runner import TextTestRunner, registerResult import warnings @@ -5,9 +6,16 @@ def defer(delay, callback, *args, **kwargs): + # Rely on late binding in case a user patches it sublime.set_timeout(lambda: callback(*args, **kwargs), delay) +AWAIT_WORKER = 'AWAIT_WORKER' +# Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user +# patches it +run_on_worker = sublime.set_timeout_async + + class DeferringTextTestRunner(TextTestRunner): """This test runner runs tests in deferred slices.""" @@ -60,6 +68,10 @@ def _continue_testing(deferred, send_value=None, throw_value=None): defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) + elif condition == AWAIT_WORKER: + run_on_worker( + partial(defer, 0, _continue_testing, deferred) + ) else: defer(10, _continue_testing, deferred) From 32420760048a93eeaf2364831cb442dc3568d1d5 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 20:42:23 +0100 Subject: [PATCH 08/15] Use partial in defer for safety and performance --- unittesting/core/st3/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index dfa3f7f1..dff1fbfc 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -7,7 +7,7 @@ def defer(delay, callback, *args, **kwargs): # Rely on late binding in case a user patches it - sublime.set_timeout(lambda: callback(*args, **kwargs), delay) + sublime.set_timeout(partial(callback, *args, **kwargs), delay) AWAIT_WORKER = 'AWAIT_WORKER' From 43cc67e947aca52e7b0018e5e9cfc084fa303b76 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Sat, 1 Dec 2018 19:17:08 +0100 Subject: [PATCH 09/15] Raise TimeoutError if condition times out --- tests/test_yield_condition.py | 13 +++++++++++++ unittesting/core/st3/runner.py | 7 +++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/test_yield_condition.py b/tests/test_yield_condition.py index 1614aeba..a6ad7e4d 100644 --- a/tests/test_yield_condition.py +++ b/tests/test_yield_condition.py @@ -15,3 +15,16 @@ def test_returns_condition_value(self): rv = yield lambda: 'Hans Peter' self.assertEqual(rv, 'Hans Peter') + + def test_handle_condition_timeout_as_failure(self): + try: + yield { + 'condition': lambda: True is False, + 'timeout': 100 + } + self.fail('Unmet condition should have thrown') + except TimeoutError as e: + self.assertEqual( + str(e), + 'Condition not fulfilled within 0.10 seconds' + ) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index dff1fbfc..c7a8ef44 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -95,8 +95,11 @@ def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=N if send_value: defer(10, _continue_testing, deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: - self.stream.writeln("Condition timeout, continue anyway.") - defer(10, _continue_testing, deferred) + error = TimeoutError( + 'Condition not fulfilled within {:.2f} seconds' + .format(timeout / 1000) + ) + defer(10, _continue_testing, deferred, throw_value=error) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) From 6857882ce47e2cd79cb24f56bab19c6c5983cbf9 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 21:07:28 +0100 Subject: [PATCH 10/15] Change timings --- tests/test_deferred_timing.py | 81 ++++++++++++++++++++++++++++++++++ unittesting/core/st3/runner.py | 25 +++++++---- 2 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 tests/test_deferred_timing.py diff --git a/tests/test_deferred_timing.py b/tests/test_deferred_timing.py new file mode 100644 index 00000000..9e802dfc --- /dev/null +++ b/tests/test_deferred_timing.py @@ -0,0 +1,81 @@ +from functools import partial +import time +from unittest.mock import patch + +import sublime + +from unittesting import DeferrableTestCase + + +def run_in_worker(fn, *args, **kwargs): + sublime.set_timeout_async(partial(fn, *args, **kwargs)) + + +# When we swap `set_timeout_async` with `set_timeout` we basically run +# our program single-threaded. +# This has some benefits: +# - We avoid async/timing issues +# - We can use plain `yield` to run Sublime's task queue empty, see below +# - Every code we run will get correct coverage +# +# However note, that Sublime will just put all async events on the queue, +# avoiding the API. We cannot patch that. That means, the event handlers +# will *not* run using plain `yield` like below, you still have to await +# them using `yield AWAIT_WORKER`. +# + +class TestTimingInDeferredTestCase(DeferrableTestCase): + + def test_a(self): + # `patch` doesn't work as a decorator with generator functions so we + # use `with` + with patch.object(sublime, 'set_timeout_async', sublime.set_timeout): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + def uut(): + run_in_worker(work, 1, 0.5) # add task (A) + run_in_worker(work, 2) # add task (B) + + uut() # after that task queue has: (A)..(B) + yield # add task (C) and wait for (C) + expected = [1, 2] + self.assertEqual(messages, expected) + + def test_b(self): + # `patch` doesn't work as a decorator with generator functions so we + # use `with` + with patch.object(sublime, 'set_timeout_async', sublime.set_timeout): + messages = [] + + def work(message, worktime=None): + if worktime: + time.sleep(worktime) + messages.append(message) + + def sub_task(): + run_in_worker(work, 2, 0.5) # add task (D) + + def uut(): + run_in_worker(work, 1, 0.3) # add task (A) + run_in_worker(sub_task) # add task (B) + + uut() + # task queue now: (A)..(B) + + yield # add task (C) and wait for (C) + # (A) runs, (B) runs and adds task (D), (C) resolves + expected = [1] + self.assertEqual(messages, expected) + # task queue now: (D) + yield # add task (E) and wait for it + # (D) runs and (E) resolves + expected = [1, 2] + self.assertEqual(messages, expected) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index c7a8ef44..8fdf3b0c 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -10,6 +10,8 @@ def defer(delay, callback, *args, **kwargs): sublime.set_timeout(partial(callback, *args, **kwargs), delay) +DEFAULT_CONDITION_POLL_TIME = 17 +DEFAULT_CONDITION_TIMEOUT = 4000 AWAIT_WORKER = 'AWAIT_WORKER' # Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user # patches it @@ -48,7 +50,7 @@ def _start_testing(): startTestRun() try: deferred = test(result) - defer(10, _continue_testing, deferred) + _continue_testing(deferred) except Exception as e: _handle_error(e) @@ -61,10 +63,10 @@ def _continue_testing(deferred, send_value=None, throw_value=None): condition = deferred.send(send_value) if callable(condition): - defer(100, _wait_condition, deferred, condition) + defer(0, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ callable(condition["condition"]): - period = condition.get("period", 100) + period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) @@ -73,7 +75,7 @@ def _continue_testing(deferred, send_value=None, throw_value=None): partial(defer, 0, _continue_testing, deferred) ) else: - defer(10, _continue_testing, deferred) + defer(0, _continue_testing, deferred) except StopIteration: _stop_testing() @@ -82,24 +84,29 @@ def _continue_testing(deferred, send_value=None, throw_value=None): except Exception as e: _handle_error(e) - def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): + def _wait_condition( + deferred, condition, + period=DEFAULT_CONDITION_POLL_TIME, + timeout=DEFAULT_CONDITION_TIMEOUT, + start_time=None + ): if start_time is None: start_time = time.time() try: send_value = condition() except Exception as e: - defer(10, _continue_testing, deferred, throw_value=e) + _continue_testing(deferred, throw_value=e) return if send_value: - defer(10, _continue_testing, deferred, send_value=send_value) + _continue_testing(deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: error = TimeoutError( 'Condition not fulfilled within {:.2f} seconds' .format(timeout / 1000) ) - defer(10, _continue_testing, deferred, throw_value=error) + _continue_testing(deferred, throw_value=error) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) @@ -157,4 +164,4 @@ def _stop_testing(): else: self.stream.write("\n") - sublime.set_timeout(_start_testing, 10) + _start_testing() From a7371cedae786421d3c9a3681e33bf071c22a743 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 21:19:54 +0100 Subject: [PATCH 11/15] Copy runner and hide it behind a 'fast_timings' flag --- unittesting/core/__init__.py | 2 + unittesting/core/st3/fast_runner.py | 167 ++++++++++++++++++++++++++++ unittesting/mixin.py | 1 + unittesting/test_package.py | 12 +- 4 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 unittesting/core/st3/fast_runner.py diff --git a/unittesting/core/__init__.py b/unittesting/core/__init__.py index 3a869e82..8d9a9ab1 100644 --- a/unittesting/core/__init__.py +++ b/unittesting/core/__init__.py @@ -1,10 +1,12 @@ from .st3.runner import DeferringTextTestRunner, AWAIT_WORKER +from .st3.fast_runner import FastDeferringTextTestRunner from .st3.case import DeferrableTestCase from .st3.suite import DeferrableTestSuite from .loader import UnitTestingLoader as TestLoader __all__ = [ "TestLoader", + "FastDeferringTextTestRunner", "DeferringTextTestRunner", "DeferrableTestCase", "DeferrableTestSuite", diff --git a/unittesting/core/st3/fast_runner.py b/unittesting/core/st3/fast_runner.py new file mode 100644 index 00000000..3785aa88 --- /dev/null +++ b/unittesting/core/st3/fast_runner.py @@ -0,0 +1,167 @@ +from functools import partial +import time +from unittest.runner import TextTestRunner, registerResult +import warnings +import sublime + + +def defer(delay, callback, *args, **kwargs): + # Rely on late binding in case a user patches it + sublime.set_timeout(partial(callback, *args, **kwargs), delay) + + +DEFAULT_CONDITION_POLL_TIME = 17 +DEFAULT_CONDITION_TIMEOUT = 4000 +AWAIT_WORKER = 'AWAIT_WORKER' +# Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user +# patches it +run_on_worker = sublime.set_timeout_async + + +class FastDeferringTextTestRunner(TextTestRunner): + """This test runner runs tests in deferred slices.""" + + def run(self, test): + """Run the given test case or test suite.""" + self.finished = False + result = self._makeResult() + registerResult(result) + result.failfast = self.failfast + result.buffer = self.buffer + startTime = time.time() + + def _start_testing(): + with warnings.catch_warnings(): + if self.warnings: + # if self.warnings is set, use it to filter all the warnings + warnings.simplefilter(self.warnings) + # if the filter is 'default' or 'always', special-case the + # warnings from the deprecated unittest methods to show them + # no more than once per module, because they can be fairly + # noisy. The -Wd and -Wa flags can be used to bypass this + # only when self.warnings is None. + if self.warnings in ['default', 'always']: + warnings.filterwarnings( + 'module', + category=DeprecationWarning, + message='Please use assert\\w+ instead.') + startTestRun = getattr(result, 'startTestRun', None) + if startTestRun is not None: + startTestRun() + try: + deferred = test(result) + _continue_testing(deferred) + + except Exception as e: + _handle_error(e) + + def _continue_testing(deferred, send_value=None, throw_value=None): + try: + if throw_value: + condition = deferred.throw(throw_value) + else: + condition = deferred.send(send_value) + + if callable(condition): + defer(0, _wait_condition, deferred, condition) + elif isinstance(condition, dict) and "condition" in condition and \ + callable(condition["condition"]): + period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) + defer(period, _wait_condition, deferred, **condition) + elif isinstance(condition, int): + defer(condition, _continue_testing, deferred) + elif condition == AWAIT_WORKER: + run_on_worker( + partial(defer, 0, _continue_testing, deferred) + ) + else: + defer(0, _continue_testing, deferred) + + except StopIteration: + _stop_testing() + self.finished = True + + except Exception as e: + _handle_error(e) + + def _wait_condition( + deferred, condition, + period=DEFAULT_CONDITION_POLL_TIME, + timeout=DEFAULT_CONDITION_TIMEOUT, + start_time=None + ): + if start_time is None: + start_time = time.time() + + try: + send_value = condition() + except Exception as e: + _continue_testing(deferred, throw_value=e) + return + + if send_value: + _continue_testing(deferred, send_value=send_value) + elif (time.time() - start_time) * 1000 >= timeout: + error = TimeoutError( + 'Condition not fulfilled within {:.2f} seconds' + .format(timeout / 1000) + ) + _continue_testing(deferred, throw_value=error) + else: + defer(period, _wait_condition, deferred, condition, period, timeout, start_time) + + def _handle_error(e): + stopTestRun = getattr(result, 'stopTestRun', None) + if stopTestRun is not None: + stopTestRun() + self.finished = True + raise e + + def _stop_testing(): + with warnings.catch_warnings(): + stopTestRun = getattr(result, 'stopTestRun', None) + if stopTestRun is not None: + stopTestRun() + + stopTime = time.time() + timeTaken = stopTime - startTime + result.printErrors() + if hasattr(result, 'separator2'): + self.stream.writeln(result.separator2) + run = result.testsRun + self.stream.writeln("Ran %d test%s in %.3fs" % + (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln() + + expectedFails = unexpectedSuccesses = skipped = 0 + try: + results = map(len, (result.expectedFailures, + result.unexpectedSuccesses, + result.skipped)) + except AttributeError: + pass + else: + expectedFails, unexpectedSuccesses, skipped = results + + infos = [] + if not result.wasSuccessful(): + self.stream.write("FAILED") + failed, errored = len(result.failures), len(result.errors) + if failed: + infos.append("failures=%d" % failed) + if errored: + infos.append("errors=%d" % errored) + else: + self.stream.write("OK") + if skipped: + infos.append("skipped=%d" % skipped) + if expectedFails: + infos.append("expected failures=%d" % expectedFails) + if unexpectedSuccesses: + infos.append("unexpected successes=%d" % unexpectedSuccesses) + if infos: + self.stream.writeln(" (%s)" % (", ".join(infos),)) + else: + self.stream.write("\n") + + _start_testing() diff --git a/unittesting/mixin.py b/unittesting/mixin.py index 2f8b1888..3b8fbcfd 100644 --- a/unittesting/mixin.py +++ b/unittesting/mixin.py @@ -17,6 +17,7 @@ "pattern": "test*.py", "async": False, "deferred": False, + "fast_timings": False, "verbosity": 2, "output": None, "reload_package_on_testing": True, diff --git a/unittesting/test_package.py b/unittesting/test_package.py index ab5f7964..f87a56fc 100644 --- a/unittesting/test_package.py +++ b/unittesting/test_package.py @@ -4,7 +4,12 @@ import os import logging from unittest import TextTestRunner, TestSuite -from .core import TestLoader, DeferringTextTestRunner, DeferrableTestCase +from .core import ( + TestLoader, + FastDeferringTextTestRunner, + DeferringTextTestRunner, + DeferrableTestCase +) from .mixin import UnitTestingMixin from .const import DONE_MESSAGE from .utils import ProgressBar, StdioSplitter @@ -62,7 +67,10 @@ def unit_testing(self, stream, package, settings, cleanup_hooks=[]): ) # use deferred test runner or default test runner if settings["deferred"]: - testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"]) + if settings["fast_timings"]: + testRunner = FastDeferringTextTestRunner(stream, verbosity=settings["verbosity"]) + else: + testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"]) else: self.verify_testsuite(tests) testRunner = TextTestRunner(stream, verbosity=settings["verbosity"]) From 57032bdfff5c2f5e3498df32057e06a39ef47b9c Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 21:21:35 +0100 Subject: [PATCH 12/15] Restore old runner --- unittesting/core/st3/runner.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index 8fdf3b0c..dff1fbfc 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -10,8 +10,6 @@ def defer(delay, callback, *args, **kwargs): sublime.set_timeout(partial(callback, *args, **kwargs), delay) -DEFAULT_CONDITION_POLL_TIME = 17 -DEFAULT_CONDITION_TIMEOUT = 4000 AWAIT_WORKER = 'AWAIT_WORKER' # Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user # patches it @@ -50,7 +48,7 @@ def _start_testing(): startTestRun() try: deferred = test(result) - _continue_testing(deferred) + defer(10, _continue_testing, deferred) except Exception as e: _handle_error(e) @@ -63,10 +61,10 @@ def _continue_testing(deferred, send_value=None, throw_value=None): condition = deferred.send(send_value) if callable(condition): - defer(0, _wait_condition, deferred, condition) + defer(100, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ callable(condition["condition"]): - period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) + period = condition.get("period", 100) defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) @@ -75,7 +73,7 @@ def _continue_testing(deferred, send_value=None, throw_value=None): partial(defer, 0, _continue_testing, deferred) ) else: - defer(0, _continue_testing, deferred) + defer(10, _continue_testing, deferred) except StopIteration: _stop_testing() @@ -84,29 +82,21 @@ def _continue_testing(deferred, send_value=None, throw_value=None): except Exception as e: _handle_error(e) - def _wait_condition( - deferred, condition, - period=DEFAULT_CONDITION_POLL_TIME, - timeout=DEFAULT_CONDITION_TIMEOUT, - start_time=None - ): + def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): if start_time is None: start_time = time.time() try: send_value = condition() except Exception as e: - _continue_testing(deferred, throw_value=e) + defer(10, _continue_testing, deferred, throw_value=e) return if send_value: - _continue_testing(deferred, send_value=send_value) + defer(10, _continue_testing, deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: - error = TimeoutError( - 'Condition not fulfilled within {:.2f} seconds' - .format(timeout / 1000) - ) - _continue_testing(deferred, throw_value=error) + self.stream.writeln("Condition timeout, continue anyway.") + defer(10, _continue_testing, deferred) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) @@ -164,4 +154,4 @@ def _stop_testing(): else: self.stream.write("\n") - _start_testing() + sublime.set_timeout(_start_testing, 10) From c8976431e6c0be4b6760630199b104e3a577d80b Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 21:23:11 +0100 Subject: [PATCH 13/15] Switch self testing to the new 'fast_timings' --- unittesting.json | 1 + 1 file changed, 1 insertion(+) diff --git a/unittesting.json b/unittesting.json index d3a15282..bbf6dff4 100644 --- a/unittesting.json +++ b/unittesting.json @@ -3,6 +3,7 @@ "pattern" : "test*.py", "async": false, "deferred": true, + "fast_timings": true, "verbosity": 2, "capture_console": false, "reload_package_on_testing": true, From bfbeb8af409e47e09fe6b530d8533610ef60521a Mon Sep 17 00:00:00 2001 From: herr kaste Date: Mon, 10 Dec 2018 23:37:12 +0100 Subject: [PATCH 14/15] Add random short delay --- tests/test_3141596.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_3141596.py b/tests/test_3141596.py index eae05a07..3ab366a6 100644 --- a/tests/test_3141596.py +++ b/tests/test_3141596.py @@ -38,7 +38,7 @@ def cleanup_package(package): def prepare_package(package, output=None, syntax_test=False, syntax_compatibility=False, - color_scheme_test=False, delay=None): + color_scheme_test=False, delay=100): def wrapper(func): @wraps(func) def real_wrapper(self): From 8cc5b96a74bcdecececd60259b800b6b9f6b97a3 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Wed, 19 Dec 2018 22:19:20 +0100 Subject: [PATCH 15/15] Name the setting 'legacy_runner' Basically, the old runner is now called LegacyDeferringTextTestRunner, and the former FastDeferringTextTestRunner is the new DeferrableTestCase. --- unittesting.json | 2 +- unittesting/core/__init__.py | 4 +- .../st3/{fast_runner.py => legacy_runner.py} | 60 +++++-------------- unittesting/core/st3/runner.py | 30 ++++++---- unittesting/mixin.py | 2 +- unittesting/test_package.py | 6 +- 6 files changed, 41 insertions(+), 63 deletions(-) rename unittesting/core/st3/{fast_runner.py => legacy_runner.py} (71%) diff --git a/unittesting.json b/unittesting.json index bbf6dff4..0b902099 100644 --- a/unittesting.json +++ b/unittesting.json @@ -3,7 +3,7 @@ "pattern" : "test*.py", "async": false, "deferred": true, - "fast_timings": true, + "legacy_runner": false, "verbosity": 2, "capture_console": false, "reload_package_on_testing": true, diff --git a/unittesting/core/__init__.py b/unittesting/core/__init__.py index 8d9a9ab1..d0c1b865 100644 --- a/unittesting/core/__init__.py +++ b/unittesting/core/__init__.py @@ -1,13 +1,13 @@ from .st3.runner import DeferringTextTestRunner, AWAIT_WORKER -from .st3.fast_runner import FastDeferringTextTestRunner +from .st3.legacy_runner import LegacyDeferringTextTestRunner from .st3.case import DeferrableTestCase from .st3.suite import DeferrableTestSuite from .loader import UnitTestingLoader as TestLoader __all__ = [ "TestLoader", - "FastDeferringTextTestRunner", "DeferringTextTestRunner", + "LegacyDeferringTextTestRunner", "DeferrableTestCase", "DeferrableTestSuite", "AWAIT_WORKER" diff --git a/unittesting/core/st3/fast_runner.py b/unittesting/core/st3/legacy_runner.py similarity index 71% rename from unittesting/core/st3/fast_runner.py rename to unittesting/core/st3/legacy_runner.py index 3785aa88..fc6cb2ff 100644 --- a/unittesting/core/st3/fast_runner.py +++ b/unittesting/core/st3/legacy_runner.py @@ -1,4 +1,3 @@ -from functools import partial import time from unittest.runner import TextTestRunner, registerResult import warnings @@ -6,19 +5,10 @@ def defer(delay, callback, *args, **kwargs): - # Rely on late binding in case a user patches it - sublime.set_timeout(partial(callback, *args, **kwargs), delay) + sublime.set_timeout(lambda: callback(*args, **kwargs), delay) -DEFAULT_CONDITION_POLL_TIME = 17 -DEFAULT_CONDITION_TIMEOUT = 4000 -AWAIT_WORKER = 'AWAIT_WORKER' -# Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user -# patches it -run_on_worker = sublime.set_timeout_async - - -class FastDeferringTextTestRunner(TextTestRunner): +class LegacyDeferringTextTestRunner(TextTestRunner): """This test runner runs tests in deferred slices.""" def run(self, test): @@ -50,32 +40,24 @@ def _start_testing(): startTestRun() try: deferred = test(result) - _continue_testing(deferred) + defer(10, _continue_testing, deferred) except Exception as e: _handle_error(e) - def _continue_testing(deferred, send_value=None, throw_value=None): + def _continue_testing(deferred): try: - if throw_value: - condition = deferred.throw(throw_value) - else: - condition = deferred.send(send_value) - + condition = next(deferred) if callable(condition): - defer(0, _wait_condition, deferred, condition) + defer(100, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ callable(condition["condition"]): - period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) + period = condition.get("period", 100) defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) - elif condition == AWAIT_WORKER: - run_on_worker( - partial(defer, 0, _continue_testing, deferred) - ) else: - defer(0, _continue_testing, deferred) + defer(10, _continue_testing, deferred) except StopIteration: _stop_testing() @@ -84,29 +66,15 @@ def _continue_testing(deferred, send_value=None, throw_value=None): except Exception as e: _handle_error(e) - def _wait_condition( - deferred, condition, - period=DEFAULT_CONDITION_POLL_TIME, - timeout=DEFAULT_CONDITION_TIMEOUT, - start_time=None - ): + def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): if start_time is None: start_time = time.time() - try: - send_value = condition() - except Exception as e: - _continue_testing(deferred, throw_value=e) - return - - if send_value: - _continue_testing(deferred, send_value=send_value) + if condition(): + defer(10, _continue_testing, deferred) elif (time.time() - start_time) * 1000 >= timeout: - error = TimeoutError( - 'Condition not fulfilled within {:.2f} seconds' - .format(timeout / 1000) - ) - _continue_testing(deferred, throw_value=error) + self.stream.writeln("Condition timeout, continue anyway.") + defer(10, _continue_testing, deferred) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) @@ -164,4 +132,4 @@ def _stop_testing(): else: self.stream.write("\n") - _start_testing() + sublime.set_timeout(_start_testing, 10) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index dff1fbfc..8fdf3b0c 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -10,6 +10,8 @@ def defer(delay, callback, *args, **kwargs): sublime.set_timeout(partial(callback, *args, **kwargs), delay) +DEFAULT_CONDITION_POLL_TIME = 17 +DEFAULT_CONDITION_TIMEOUT = 4000 AWAIT_WORKER = 'AWAIT_WORKER' # Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user # patches it @@ -48,7 +50,7 @@ def _start_testing(): startTestRun() try: deferred = test(result) - defer(10, _continue_testing, deferred) + _continue_testing(deferred) except Exception as e: _handle_error(e) @@ -61,10 +63,10 @@ def _continue_testing(deferred, send_value=None, throw_value=None): condition = deferred.send(send_value) if callable(condition): - defer(100, _wait_condition, deferred, condition) + defer(0, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ callable(condition["condition"]): - period = condition.get("period", 100) + period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) @@ -73,7 +75,7 @@ def _continue_testing(deferred, send_value=None, throw_value=None): partial(defer, 0, _continue_testing, deferred) ) else: - defer(10, _continue_testing, deferred) + defer(0, _continue_testing, deferred) except StopIteration: _stop_testing() @@ -82,21 +84,29 @@ def _continue_testing(deferred, send_value=None, throw_value=None): except Exception as e: _handle_error(e) - def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): + def _wait_condition( + deferred, condition, + period=DEFAULT_CONDITION_POLL_TIME, + timeout=DEFAULT_CONDITION_TIMEOUT, + start_time=None + ): if start_time is None: start_time = time.time() try: send_value = condition() except Exception as e: - defer(10, _continue_testing, deferred, throw_value=e) + _continue_testing(deferred, throw_value=e) return if send_value: - defer(10, _continue_testing, deferred, send_value=send_value) + _continue_testing(deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: - self.stream.writeln("Condition timeout, continue anyway.") - defer(10, _continue_testing, deferred) + error = TimeoutError( + 'Condition not fulfilled within {:.2f} seconds' + .format(timeout / 1000) + ) + _continue_testing(deferred, throw_value=error) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) @@ -154,4 +164,4 @@ def _stop_testing(): else: self.stream.write("\n") - sublime.set_timeout(_start_testing, 10) + _start_testing() diff --git a/unittesting/mixin.py b/unittesting/mixin.py index 3b8fbcfd..8df5d2a0 100644 --- a/unittesting/mixin.py +++ b/unittesting/mixin.py @@ -17,7 +17,7 @@ "pattern": "test*.py", "async": False, "deferred": False, - "fast_timings": False, + "legacy_runner": True, "verbosity": 2, "output": None, "reload_package_on_testing": True, diff --git a/unittesting/test_package.py b/unittesting/test_package.py index f87a56fc..56cf2f8a 100644 --- a/unittesting/test_package.py +++ b/unittesting/test_package.py @@ -6,8 +6,8 @@ from unittest import TextTestRunner, TestSuite from .core import ( TestLoader, - FastDeferringTextTestRunner, DeferringTextTestRunner, + LegacyDeferringTextTestRunner, DeferrableTestCase ) from .mixin import UnitTestingMixin @@ -67,8 +67,8 @@ def unit_testing(self, stream, package, settings, cleanup_hooks=[]): ) # use deferred test runner or default test runner if settings["deferred"]: - if settings["fast_timings"]: - testRunner = FastDeferringTextTestRunner(stream, verbosity=settings["verbosity"]) + if settings["legacy_runner"]: + testRunner = LegacyDeferringTextTestRunner(stream, verbosity=settings["verbosity"]) else: testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"]) else: