Deterministic timing for DeferrableTestCase

tests/test_await_worker.py

@@ -0,0 +1,47 @@
+from functools import partial
+import time
+
+import sublime
+
+from unittesting import DeferrableTestCase, AWAIT_WORKER
+
+
+def run_in_worker(fn, *args, **kwargs):
+    sublime.set_timeout_async(partial(fn, *args, **kwargs))
+
+
+class TestAwaitingWorkerInDeferredTestCase(DeferrableTestCase):
+
+    def test_ensure_plain_yield_is_faster_than_the_worker_thread(self):
+        messages = []
+
+        def work(message, worktime=None):
+            # simulate that a task might take some time
+            # this will not yield back but block
+            if worktime:
+                time.sleep(worktime)
+
+            messages.append(message)
+
+        run_in_worker(work, 1, 5)
+
+        yield
+
+        self.assertEqual(messages, [])
+
+    def test_await_worker(self):
+        messages = []
+
+        def work(message, worktime=None):
+            # simulate that a task might take some time
+            # this will not yield back but block
+            if worktime:
+                time.sleep(worktime)
+
+            messages.append(message)
+
+        run_in_worker(work, 1, 0.5)
+
+        yield AWAIT_WORKER
+
+        self.assertEqual(messages, [1])

tests/test_deferred_timing.py

@@ -0,0 +1,81 @@
+from functools import partial
+import time
+from unittest.mock import patch
+
+import sublime
+
+from unittesting import DeferrableTestCase
+
+
+def run_in_worker(fn, *args, **kwargs):
+    sublime.set_timeout_async(partial(fn, *args, **kwargs))
+
+
+# When we swap `set_timeout_async` with `set_timeout` we basically run
+# our program single-threaded.
+# This has some benefits:
+# - We avoid async/timing issues
+# - We can use plain `yield` to run Sublime's task queue empty, see below
+# - Every code we run will get correct coverage
+#
+# However note, that Sublime will just put all async events on the queue,
+# avoiding the API. We cannot patch that. That means, the event handlers
+# will *not* run using plain `yield` like below, you still have to await
+# them using `yield AWAIT_WORKER`.
+#
+
+class TestTimingInDeferredTestCase(DeferrableTestCase):
+
+    def test_a(self):
+        # `patch` doesn't work as a decorator with generator functions so we
+        # use `with`
+        with patch.object(sublime, 'set_timeout_async', sublime.set_timeout):
+            messages = []
+
+            def work(message, worktime=None):
+                # simulate that a task might take some time
+                # this will not yield back but block
+                if worktime:
+                    time.sleep(worktime)
+
+                messages.append(message)
+
+            def uut():
+                run_in_worker(work, 1, 0.5)   # add task (A)
+                run_in_worker(work, 2)        # add task (B)
+
+            uut()   # after that task queue has: (A)..(B)
+            yield   # add task (C) and wait for (C)
+            expected = [1, 2]
+            self.assertEqual(messages, expected)
+
+    def test_b(self):
+        # `patch` doesn't work as a decorator with generator functions so we
+        # use `with`
+        with patch.object(sublime, 'set_timeout_async', sublime.set_timeout):
+            messages = []
+
+            def work(message, worktime=None):
+                if worktime:
+                    time.sleep(worktime)
+                messages.append(message)
+
+            def sub_task():
+                run_in_worker(work, 2, 0.5)  # add task (D)
+
+            def uut():
+                run_in_worker(work, 1, 0.3)    # add task (A)
+                run_in_worker(sub_task)      # add task (B)
+
+            uut()
+            # task queue now: (A)..(B)
+
+            yield  # add task (C) and wait for (C)
+            #        (A) runs, (B) runs and adds task (D), (C) resolves
+            expected = [1]
+            self.assertEqual(messages, expected)
+            # task queue now: (D)
+            yield  # add task (E) and wait for it
+            #        (D) runs and (E) resolves
+            expected = [1, 2]
+            self.assertEqual(messages, expected)

unittesting/__init__.py

@@ -1,4 +1,4 @@
-from .core import DeferrableTestCase
+from .core import DeferrableTestCase, AWAIT_WORKER
 from .scheduler import UnitTestingRunSchedulerCommand
 from .scheduler import run_scheduler
 from .test_package import UnitTestingCommand
@@ -22,5 +22,6 @@
     "UnitTestingCurrentPackageCoverageCommand",
     "UnitTestingSyntaxCommand",
     "UnitTestingSyntaxCompatibilityCommand",
-    "UnitTestingColorSchemeCommand"
+    "UnitTestingColorSchemeCommand",
+    "AWAIT_WORKER"
 ]

unittesting/core/__init__.py

@@ -1,6 +1,12 @@
-from .st3.runner import DeferringTextTestRunner
+from .st3.runner import DeferringTextTestRunner, AWAIT_WORKER
 from .st3.case import DeferrableTestCase
 from .st3.suite import DeferrableTestSuite
 from .loader import UnitTestingLoader as TestLoader
 
-__all__ = ["TestLoader", "DeferringTextTestRunner", "DeferrableTestCase", "DeferrableTestSuite"]
+__all__ = [
+    "TestLoader",
+    "DeferringTextTestRunner",
+    "DeferrableTestCase",
+    "DeferrableTestSuite",
+    "AWAIT_WORKER"
+]

unittesting/core/st3/runner.py

@@ -1,11 +1,19 @@
+from functools import partial
 import time
 from unittest.runner import TextTestRunner, registerResult
 import warnings
 import sublime
 
 
 def defer(delay, callback, *args, **kwargs):
-    sublime.set_timeout(lambda: callback(*args, **kwargs), delay)
+    # Rely on late binding in case a user patches it
+    sublime.set_timeout(partial(callback, *args, **kwargs), delay)
+
+
+AWAIT_WORKER = 'AWAIT_WORKER'
+# Extract `set_timeout_async`, t.i. *avoid* late binding, in case a user
+# patches it
+run_on_worker = sublime.set_timeout_async
 
 
 class DeferringTextTestRunner(TextTestRunner):
@@ -40,7 +48,7 @@ def _start_testing():
                     startTestRun()
                 try:
                     deferred = test(result)
-                    defer(10, _continue_testing, deferred)
+                    _continue_testing(deferred)
 
                 except Exception as e:
                     _handle_error(e)
@@ -49,15 +57,19 @@ def _continue_testing(deferred):
             try:
                 condition = next(deferred)
                 if callable(condition):
-                    defer(100, _wait_condition, deferred, condition)
+                    defer(0, _wait_condition, deferred, condition)
                 elif isinstance(condition, dict) and "condition" in condition and \
                         callable(condition["condition"]):
-                    period = condition.get("period", 100)
+                    period = condition.get("period", 5)
                     defer(period, _wait_condition, deferred, **condition)
                 elif isinstance(condition, int):
                     defer(condition, _continue_testing, deferred)
+                elif condition == AWAIT_WORKER:
+                    run_on_worker(
+                        partial(defer, 0, _continue_testing, deferred)
+                    )
                 else:
-                    defer(10, _continue_testing, deferred)
+                    defer(0, _continue_testing, deferred)
 
             except StopIteration:
                 _stop_testing()
@@ -66,15 +78,15 @@ def _continue_testing(deferred):
             except Exception as e:
                 _handle_error(e)
 
-        def _wait_condition(deferred, condition, period=100, timeout=10000, start_time=None):
+        def _wait_condition(deferred, condition, period=5, timeout=10000, start_time=None):
             if start_time is None:
                 start_time = time.time()
 
             if condition():
-                defer(10, _continue_testing, deferred)
+                _continue_testing(deferred)
             elif (time.time() - start_time) * 1000 >= timeout:
                 self.stream.writeln("Condition timeout, continue anyway.")
-                defer(10, _continue_testing, deferred)
+                _continue_testing(deferred)
             else:
                 defer(period, _wait_condition, deferred, condition, period, timeout, start_time)
 
@@ -132,4 +144,4 @@ def _stop_testing():
             else:
                 self.stream.write("\n")
 
-        sublime.set_timeout(_start_testing, 10)
+        _start_testing()