diff --git a/.flake8 b/.flake8 index 1d15de50..8a8ea62b 100644 --- a/.flake8 +++ b/.flake8 @@ -7,6 +7,7 @@ max-line-length = 120 # D103 Missing docstring in public function # D104 Missing docstring in public package # D107 Missing docstring in __init__ +# W503 line break before binary operator # W504 line break after binary operator # W606 'async' and 'await' are reserved keywords starting with Python 3.7 -ignore = D100, D101, D102, D103, D104, D107, W504, W606 +ignore = D100, D101, D102, D103, D104, D107, W503, W504, W606 diff --git a/tests/test_3141596.py b/tests/test_3141596.py index eae05a07..3ab366a6 100644 --- a/tests/test_3141596.py +++ b/tests/test_3141596.py @@ -38,7 +38,7 @@ def cleanup_package(package): def prepare_package(package, output=None, syntax_test=False, syntax_compatibility=False, - color_scheme_test=False, delay=None): + color_scheme_test=False, delay=100): def wrapper(func): @wraps(func) def real_wrapper(self): diff --git a/tests/test_await_worker.py b/tests/test_await_worker.py new file mode 100644 index 00000000..866d34da --- /dev/null +++ b/tests/test_await_worker.py @@ -0,0 +1,47 @@ +from functools import partial +import time + +import sublime + +from unittesting import DeferrableTestCase, AWAIT_WORKER + + +def run_in_worker(fn, *args, **kwargs): + sublime.set_timeout_async(partial(fn, *args, **kwargs)) + + +class TestAwaitingWorkerInDeferredTestCase(DeferrableTestCase): + + def test_ensure_plain_yield_is_faster_than_the_worker_thread(self): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + run_in_worker(work, 1, 5) + + yield + + self.assertEqual(messages, []) + + def test_await_worker(self): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + run_in_worker(work, 1, 0.5) + + yield AWAIT_WORKER + + self.assertEqual(messages, [1]) diff --git a/tests/test_deferred_timing.py b/tests/test_deferred_timing.py new file mode 100644 index 00000000..9e802dfc --- /dev/null +++ b/tests/test_deferred_timing.py @@ -0,0 +1,81 @@ +from functools import partial +import time +from unittest.mock import patch + +import sublime + +from unittesting import DeferrableTestCase + + +def run_in_worker(fn, *args, **kwargs): + sublime.set_timeout_async(partial(fn, *args, **kwargs)) + + +# When we swap `set_timeout_async` with `set_timeout` we basically run +# our program single-threaded. +# This has some benefits: +# - We avoid async/timing issues +# - We can use plain `yield` to run Sublime's task queue empty, see below +# - Every code we run will get correct coverage +# +# However note, that Sublime will just put all async events on the queue, +# avoiding the API. We cannot patch that. That means, the event handlers +# will *not* run using plain `yield` like below, you still have to await +# them using `yield AWAIT_WORKER`. +# + +class TestTimingInDeferredTestCase(DeferrableTestCase): + + def test_a(self): + # `patch` doesn't work as a decorator with generator functions so we + # use `with` + with patch.object(sublime, 'set_timeout_async', sublime.set_timeout): + messages = [] + + def work(message, worktime=None): + # simulate that a task might take some time + # this will not yield back but block + if worktime: + time.sleep(worktime) + + messages.append(message) + + def uut(): + run_in_worker(work, 1, 0.5) # add task (A) + run_in_worker(work, 2) # add task (B) + + uut() # after that task queue has: (A)..(B) + yield # add task (C) and wait for (C) + expected = [1, 2] + self.assertEqual(messages, expected) + + def test_b(self): + # `patch` doesn't work as a decorator with generator functions so we + # use `with` + with patch.object(sublime, 'set_timeout_async', sublime.set_timeout): + messages = [] + + def work(message, worktime=None): + if worktime: + time.sleep(worktime) + messages.append(message) + + def sub_task(): + run_in_worker(work, 2, 0.5) # add task (D) + + def uut(): + run_in_worker(work, 1, 0.3) # add task (A) + run_in_worker(sub_task) # add task (B) + + uut() + # task queue now: (A)..(B) + + yield # add task (C) and wait for (C) + # (A) runs, (B) runs and adds task (D), (C) resolves + expected = [1] + self.assertEqual(messages, expected) + # task queue now: (D) + yield # add task (E) and wait for it + # (D) runs and (E) resolves + expected = [1, 2] + self.assertEqual(messages, expected) diff --git a/tests/test_ensure_do_cleanups.py b/tests/test_ensure_do_cleanups.py new file mode 100644 index 00000000..6a827bd0 --- /dev/null +++ b/tests/test_ensure_do_cleanups.py @@ -0,0 +1,26 @@ +from unittesting import DeferrableTestCase + + +class TestExplicitDoCleanups(DeferrableTestCase): + + def test_manually_calling_do_cleanups_works(self): + messages = [] + + def work(message): + messages.append(message) + + self.addCleanup(work, 1) + yield from self.doCleanups() + + self.assertEqual(messages, [1]) + + +cleanup_called = [] + + +class TestImplicitDoCleanupsOnTeardown(DeferrableTestCase): + def test_a_prepare(self): + self.addCleanup(lambda: cleanup_called.append(1)) + + def test_b_assert(self): + self.assertEqual(cleanup_called, [1]) diff --git a/tests/test_yield_condition.py b/tests/test_yield_condition.py new file mode 100644 index 00000000..a6ad7e4d --- /dev/null +++ b/tests/test_yield_condition.py @@ -0,0 +1,30 @@ +from unittesting import DeferrableTestCase + + +class TestYieldConditionsHandlingInDeferredTestCase(DeferrableTestCase): + def test_reraises_errors_raised_in_conditions(self): + try: + yield lambda: 1 / 0 + self.fail('Did not reraise the exception from the condition') + except ZeroDivisionError: + pass + except Exception: + self.fail('Did not throw the original exception') + + def test_returns_condition_value(self): + rv = yield lambda: 'Hans Peter' + + self.assertEqual(rv, 'Hans Peter') + + def test_handle_condition_timeout_as_failure(self): + try: + yield { + 'condition': lambda: True is False, + 'timeout': 100 + } + self.fail('Unmet condition should have thrown') + except TimeoutError as e: + self.assertEqual( + str(e), + 'Condition not fulfilled within 0.10 seconds' + ) diff --git a/unittesting.json b/unittesting.json index d3a15282..0b902099 100644 --- a/unittesting.json +++ b/unittesting.json @@ -3,6 +3,7 @@ "pattern" : "test*.py", "async": false, "deferred": true, + "legacy_runner": false, "verbosity": 2, "capture_console": false, "reload_package_on_testing": true, diff --git a/unittesting/__init__.py b/unittesting/__init__.py index 7648f383..3c2108d6 100644 --- a/unittesting/__init__.py +++ b/unittesting/__init__.py @@ -1,4 +1,4 @@ -from .core import DeferrableTestCase +from .core import DeferrableTestCase, AWAIT_WORKER from .scheduler import UnitTestingRunSchedulerCommand from .scheduler import run_scheduler from .test_package import UnitTestingCommand @@ -22,5 +22,6 @@ "UnitTestingCurrentPackageCoverageCommand", "UnitTestingSyntaxCommand", "UnitTestingSyntaxCompatibilityCommand", - "UnitTestingColorSchemeCommand" + "UnitTestingColorSchemeCommand", + "AWAIT_WORKER" ] diff --git a/unittesting/core/__init__.py b/unittesting/core/__init__.py index 73c6b6a0..d0c1b865 100644 --- a/unittesting/core/__init__.py +++ b/unittesting/core/__init__.py @@ -1,6 +1,14 @@ -from .st3.runner import DeferringTextTestRunner +from .st3.runner import DeferringTextTestRunner, AWAIT_WORKER +from .st3.legacy_runner import LegacyDeferringTextTestRunner from .st3.case import DeferrableTestCase from .st3.suite import DeferrableTestSuite from .loader import UnitTestingLoader as TestLoader -__all__ = ["TestLoader", "DeferringTextTestRunner", "DeferrableTestCase", "DeferrableTestSuite"] +__all__ = [ + "TestLoader", + "DeferringTextTestRunner", + "LegacyDeferringTextTestRunner", + "DeferrableTestCase", + "DeferrableTestSuite", + "AWAIT_WORKER" +] diff --git a/unittesting/core/st3/case.py b/unittesting/core/st3/case.py index 9d52e8b5..29e4123c 100644 --- a/unittesting/core/st3/case.py +++ b/unittesting/core/st3/case.py @@ -63,18 +63,12 @@ def run(self, result=None): outcome = _Outcome() self._outcomeForDoCleanups = outcome - deferred = self._executeTestPart(self.setUp, outcome) - if isiterable(deferred): - yield from deferred + yield from self._executeTestPart(self.setUp, outcome) if outcome.success: - deferred = self._executeTestPart(testMethod, outcome, isTest=True) - if isiterable(deferred): - yield from deferred - deferred = self._executeTestPart(self.tearDown, outcome) - if isiterable(deferred): - yield from deferred + yield from self._executeTestPart(testMethod, outcome, isTest=True) + yield from self._executeTestPart(self.tearDown, outcome) - self.doCleanups() + yield from self.doCleanups() if outcome.success: result.addSuccess(self) else: @@ -110,3 +104,18 @@ def run(self, result=None): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() + + def doCleanups(self): + """Execute all cleanup functions. + + Normally called for you after tearDown. + """ + outcome = self._outcomeForDoCleanups or _Outcome() + while self._cleanups: + function, args, kwargs = self._cleanups.pop() + part = lambda: function(*args, **kwargs) # noqa: E731 + yield from self._executeTestPart(part, outcome) + + # return this for backwards compatibility + # even though we no longer us it internally + return outcome.success diff --git a/unittesting/core/st3/legacy_runner.py b/unittesting/core/st3/legacy_runner.py new file mode 100644 index 00000000..fc6cb2ff --- /dev/null +++ b/unittesting/core/st3/legacy_runner.py @@ -0,0 +1,135 @@ +import time +from unittest.runner import TextTestRunner, registerResult +import warnings +import sublime + + +def defer(delay, callback, *args, **kwargs): + sublime.set_timeout(lambda: callback(*args, **kwargs), delay) + + +class LegacyDeferringTextTestRunner(TextTestRunner): + """This test runner runs tests in deferred slices.""" + + def run(self, test): + """Run the given test case or test suite.""" + self.finished = False + result = self._makeResult() + registerResult(result) + result.failfast = self.failfast + result.buffer = self.buffer + startTime = time.time() + + def _start_testing(): + with warnings.catch_warnings(): + if self.warnings: + # if self.warnings is set, use it to filter all the warnings + warnings.simplefilter(self.warnings) + # if the filter is 'default' or 'always', special-case the + # warnings from the deprecated unittest methods to show them + # no more than once per module, because they can be fairly + # noisy. The -Wd and -Wa flags can be used to bypass this + # only when self.warnings is None. + if self.warnings in ['default', 'always']: + warnings.filterwarnings( + 'module', + category=DeprecationWarning, + message='Please use assert\\w+ instead.') + startTestRun = getattr(result, 'startTestRun', None) + if startTestRun is not None: + startTestRun() + try: + deferred = test(result) + defer(10, _continue_testing, deferred) + + except Exception as e: + _handle_error(e) + + def _continue_testing(deferred): + try: + condition = next(deferred) + if callable(condition): + defer(100, _wait_condition, deferred, condition) + elif isinstance(condition, dict) and "condition" in condition and \ + callable(condition["condition"]): + period = condition.get("period", 100) + defer(period, _wait_condition, deferred, **condition) + elif isinstance(condition, int): + defer(condition, _continue_testing, deferred) + else: + defer(10, _continue_testing, deferred) + + except StopIteration: + _stop_testing() + self.finished = True + + except Exception as e: + _handle_error(e) + + def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): + if start_time is None: + start_time = time.time() + + if condition(): + defer(10, _continue_testing, deferred) + elif (time.time() - start_time) * 1000 >= timeout: + self.stream.writeln("Condition timeout, continue anyway.") + defer(10, _continue_testing, deferred) + else: + defer(period, _wait_condition, deferred, condition, period, timeout, start_time) + + def _handle_error(e): + stopTestRun = getattr(result, 'stopTestRun', None) + if stopTestRun is not None: + stopTestRun() + self.finished = True + raise e + + def _stop_testing(): + with warnings.catch_warnings(): + stopTestRun = getattr(result, 'stopTestRun', None) + if stopTestRun is not None: + stopTestRun() + + stopTime = time.time() + timeTaken = stopTime - startTime + result.printErrors() + if hasattr(result, 'separator2'): + self.stream.writeln(result.separator2) + run = result.testsRun + self.stream.writeln("Ran %d test%s in %.3fs" % + (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln() + + expectedFails = unexpectedSuccesses = skipped = 0 + try: + results = map(len, (result.expectedFailures, + result.unexpectedSuccesses, + result.skipped)) + except AttributeError: + pass + else: + expectedFails, unexpectedSuccesses, skipped = results + + infos = [] + if not result.wasSuccessful(): + self.stream.write("FAILED") + failed, errored = len(result.failures), len(result.errors) + if failed: + infos.append("failures=%d" % failed) + if errored: + infos.append("errors=%d" % errored) + else: + self.stream.write("OK") + if skipped: + infos.append("skipped=%d" % skipped) + if expectedFails: + infos.append("expected failures=%d" % expectedFails) + if unexpectedSuccesses: + infos.append("unexpected successes=%d" % unexpectedSuccesses) + if infos: + self.stream.writeln(" (%s)" % (", ".join(infos),)) + else: + self.stream.write("\n") + + sublime.set_timeout(_start_testing, 10) diff --git a/unittesting/core/st3/runner.py b/unittesting/core/st3/runner.py index 89623f86..8fdf3b0c 100644 --- a/unittesting/core/st3/runner.py +++ b/unittesting/core/st3/runner.py @@ -1,3 +1,4 @@ +from functools import partial import time from unittest.runner import TextTestRunner, registerResult import warnings @@ -5,7 +6,16 @@ def defer(delay, callback, *args, **kwargs): - sublime.set_timeout(lambda: callback(*args, **kwargs), delay) + # Rely on late binding in case a user patches it + sublime.set_timeout(partial(callback, *args, **kwargs), delay) + + +DEFAULT_CONDITION_POLL_TIME = 17 +DEFAULT_CONDITION_TIMEOUT = 4000 +AWAIT_WORKER = 'AWAIT_WORKER' +# Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user +# patches it +run_on_worker = sublime.set_timeout_async class DeferringTextTestRunner(TextTestRunner): @@ -40,24 +50,32 @@ def _start_testing(): startTestRun() try: deferred = test(result) - defer(10, _continue_testing, deferred) + _continue_testing(deferred) except Exception as e: _handle_error(e) - def _continue_testing(deferred): + def _continue_testing(deferred, send_value=None, throw_value=None): try: - condition = next(deferred) + if throw_value: + condition = deferred.throw(throw_value) + else: + condition = deferred.send(send_value) + if callable(condition): - defer(100, _wait_condition, deferred, condition) + defer(0, _wait_condition, deferred, condition) elif isinstance(condition, dict) and "condition" in condition and \ callable(condition["condition"]): - period = condition.get("period", 100) + period = condition.get("period", DEFAULT_CONDITION_POLL_TIME) defer(period, _wait_condition, deferred, **condition) elif isinstance(condition, int): defer(condition, _continue_testing, deferred) + elif condition == AWAIT_WORKER: + run_on_worker( + partial(defer, 0, _continue_testing, deferred) + ) else: - defer(10, _continue_testing, deferred) + defer(0, _continue_testing, deferred) except StopIteration: _stop_testing() @@ -66,15 +84,29 @@ def _continue_testing(deferred): except Exception as e: _handle_error(e) - def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None): + def _wait_condition( + deferred, condition, + period=DEFAULT_CONDITION_POLL_TIME, + timeout=DEFAULT_CONDITION_TIMEOUT, + start_time=None + ): if start_time is None: start_time = time.time() - if condition(): - defer(10, _continue_testing, deferred) + try: + send_value = condition() + except Exception as e: + _continue_testing(deferred, throw_value=e) + return + + if send_value: + _continue_testing(deferred, send_value=send_value) elif (time.time() - start_time) * 1000 >= timeout: - self.stream.writeln("Condition timeout, continue anyway.") - defer(10, _continue_testing, deferred) + error = TimeoutError( + 'Condition not fulfilled within {:.2f} seconds' + .format(timeout / 1000) + ) + _continue_testing(deferred, throw_value=error) else: defer(period, _wait_condition, deferred, condition, period, timeout, start_time) @@ -132,4 +164,4 @@ def _stop_testing(): else: self.stream.write("\n") - sublime.set_timeout(_start_testing, 10) + _start_testing() diff --git a/unittesting/mixin.py b/unittesting/mixin.py index ca9bd43f..1e051d2f 100644 --- a/unittesting/mixin.py +++ b/unittesting/mixin.py @@ -17,6 +17,7 @@ "pattern": "test*.py", "async": False, "deferred": False, + "legacy_runner": True, "verbosity": 2, "output": None, "reload_package_on_testing": True, @@ -62,13 +63,6 @@ def current_package_name(self): return None - @property - def current_test_file(self): - current_file = sublime.active_window().active_view().file_name() - if current_file and current_file.endswith(".py"): - current_file = os.path.basename(current_file) - return current_file - def input_parser(self, package): m = re.match(r'([^:]+):(.+)', package) if m: diff --git a/unittesting/test_current.py b/unittesting/test_current.py index 16a321ea..6a1e01b9 100644 --- a/unittesting/test_current.py +++ b/unittesting/test_current.py @@ -1,3 +1,5 @@ +from fnmatch import fnmatch +import os import sublime import sys from .test_package import UnitTestingCommand @@ -43,9 +45,23 @@ def run(self, **kwargs): sublime.message_dialog("Cannot determine package name.") return - test_file = self.current_test_file - if not test_file: - test_file = "" + window = sublime.active_window() + if not window: + return + + view = window.active_view() + + settings = self.load_unittesting_settings(project_name, kwargs) + current_file = (view and view.file_name()) or '' + file_name = os.path.basename(current_file) + if file_name and fnmatch(file_name, settings['pattern']): + test_file = file_name + window.settings().set('UnitTesting.last_test_file', test_file) + else: + test_file = ( + window.settings().get('UnitTesting.last_test_file') + or current_file + ) sublime.set_timeout_async( lambda: super(UnitTestingCurrentFileCommand, self).run( diff --git a/unittesting/test_package.py b/unittesting/test_package.py index ab5f7964..56cf2f8a 100644 --- a/unittesting/test_package.py +++ b/unittesting/test_package.py @@ -4,7 +4,12 @@ import os import logging from unittest import TextTestRunner, TestSuite -from .core import TestLoader, DeferringTextTestRunner, DeferrableTestCase +from .core import ( + TestLoader, + DeferringTextTestRunner, + LegacyDeferringTextTestRunner, + DeferrableTestCase +) from .mixin import UnitTestingMixin from .const import DONE_MESSAGE from .utils import ProgressBar, StdioSplitter @@ -62,7 +67,10 @@ def unit_testing(self, stream, package, settings, cleanup_hooks=[]): ) # use deferred test runner or default test runner if settings["deferred"]: - testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"]) + if settings["legacy_runner"]: + testRunner = LegacyDeferringTextTestRunner(stream, verbosity=settings["verbosity"]) + else: + testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"]) else: self.verify_testsuite(tests) testRunner = TextTestRunner(stream, verbosity=settings["verbosity"])