Add UnittestWorker for tests using unittest

Instead of parsing the output of unittest, which is supposed to be
human reading and thus error prone to parse, run the tests
programmatically and send the test results over a separate channel.
This mirrors what we do with pytest.

spyder_unittest/backend/tests/test_unittestrunner.py

@@ -6,255 +6,176 @@
 """Tests for unittestrunner.py"""
 
 # Standard library imports
+import os.path as osp
 import sys
+from unittest.mock import Mock
 
 # Local imports
-from spyder_unittest.backend.runnerbase import Category
 from spyder_unittest.backend.unittestrunner import UnittestRunner
+from spyder_unittest.backend.runnerbase import Category, TestResult
+from spyder_unittest.widgets.configdialog import Config
+
+
+def test_unittestrunner_create_argument_list(monkeypatch):
+    """
+    Test that UnittestRunner.createArgumentList() returns the expected list.
+    """
+    config = Config()
+    cov_path = None
+    MockZMQStreamReader = Mock()
+    monkeypatch.setattr(
+        'spyder_unittest.backend.unittestrunner.ZmqStreamReader',
+        MockZMQStreamReader)
+    mock_reader = MockZMQStreamReader()
+    mock_reader.port = 42
+    runner = UnittestRunner(None, 'resultfile')
+    runner.reader = mock_reader
+    monkeypatch.setattr(
+        'spyder_unittest.backend.unittestrunner.osp.dirname',
+        lambda _: 'dir')
+
+    result = runner.create_argument_list(config, cov_path)
+
+    pyfile = osp.join('dir', 'workers', 'unittestworker.py')
+    assert result == [pyfile, '42']
+
+
+def test_unittestrunner_start(monkeypatch):
+    """
+    Test that UnittestRunner.start() sets the .config and .reader members
+    correctly, that it connects to the reader's sig_received, and that it
+    called the base class method.
+    """
+    MockZMQStreamReader = Mock()
+    monkeypatch.setattr(
+        'spyder_unittest.backend.unittestrunner.ZmqStreamReader',
+        MockZMQStreamReader)
+    mock_reader = MockZMQStreamReader()
+    mock_base_start = Mock()
+    monkeypatch.setattr('spyder_unittest.backend.unittestrunner.RunnerBase.start',
+                        mock_base_start)
+    runner = UnittestRunner(None, 'results')
+    config = Config()
+    cov_path = None
+
+    runner.start(config, cov_path, sys.executable, ['pythondir'])
+
+    assert runner.config is config
+    assert runner.reader is mock_reader
+    runner.reader.sig_received.connect.assert_called_once_with(
+        runner.process_output)
+    mock_base_start.assert_called_once_with(
+        config, cov_path, sys.executable, ['pythondir'])
+
+
+def test_unittestrunner_process_output_with_collected(qtbot):
+    """Test UnittestRunner.processOutput() with two `collected` events."""
+    runner = UnittestRunner(None)
+    output = [{'event': 'collected', 'id': 'spam.ham'},
+              {'event': 'collected', 'id': 'eggs.bacon'}]
 
+    with qtbot.waitSignal(runner.sig_collected) as blocker:
+        runner.process_output(output)
 
-# Up to Python 3.10, unittest output read:
-# test_fail (testing.test_unittest.MyTest) ... FAIL
-# but from Python 3.11, it reads:
-# test_fail (testing.test_unittest.MyTest.test_fail) ... FAIL
-# These tests only test the system executable; they do not test
-# the situation where the requested interpreter is different.
-IS_PY311_OR_GREATER = sys.version_info[:2] >= (3, 11)
+    expected = ['spam.ham', 'eggs.bacon']
+    assert blocker.args == [expected]
 
 
-def test_unittestrunner_load_data_with_two_tests():
-    output10 = """test_isupper (teststringmethods.TestStringMethods) ... ok
-test_split (teststringmethods.TestStringMethods) ... ok
+def test_unittestrunner_process_output_with_starttest(qtbot):
+    """Test UnittestRunner.processOutput() with two `startTest` events."""
+    runner = UnittestRunner(None)
+    output = [{'event': 'startTest', 'id': 'spam.ham'},
+              {'event': 'startTest', 'id': 'eggs.bacon'}]
 
-----------------------------------------------------------------------
-Ran 2 tests in 0.012s
+    with qtbot.waitSignal(runner.sig_starttest) as blocker:
+        runner.process_output(output)
 
-OK
-"""
-    output11 = """test_isupper (teststringmethods.TestStringMethods.test_isupper) ... ok
-test_split (teststringmethods.TestStringMethods.test_split) ... ok
+    expected = ['spam.ham', 'eggs.bacon']
+    assert blocker.args == [expected]
 
-----------------------------------------------------------------------
-Ran 2 tests in 0.012s
 
-OK
-"""
-    output = output11 if IS_PY311_OR_GREATER else output10
+def test_unittestrunner_process_output_with_addsuccess(qtbot):
+    """Test UnittestRunner.processOutput() with an `addSuccess` event."""
     runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    res = runner.load_data(output)
-    assert len(res) == 2
+    output = [{'event': 'addSuccess', 'id': 'spam.ham'}]
 
-    assert res[0].category == Category.OK
-    assert res[0].status == 'ok'
-    assert res[0].name == 'teststringmethods.TestStringMethods.test_isupper'
-    assert res[0].message == ''
-    assert res[0].extra_text == []
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
 
-    assert res[1].category == Category.OK
-    assert res[1].status == 'ok'
-    assert res[1].name == 'teststringmethods.TestStringMethods.test_split'
-    assert res[1].message == ''
-    assert res[1].extra_text == []
+    expected = [TestResult(Category.OK, 'success', 'spam.ham')]
+    assert blocker.args == [expected]
 
 
-def test_unittestrunner_load_data_with_one_test():
-    output10 = """test1 (test_foo.Bar) ... ok
+def test_unittestrunner_process_output_with_addfailure(qtbot):
+    """Test UnittestRunner.processOutput() with an `addFailure` event."""
+    runner = UnittestRunner(None)
+    output = [{'event': 'addFailure',
+               'id': 'spam.ham',
+               'reason': 'exception',
+               'err': 'traceback'}]
 
-----------------------------------------------------------------------
-Ran 1 test in 0.000s
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
 
-OK
-"""
-    output11 = """test1 (test_foo.Bar.test1) ... ok
+    expected = [TestResult(Category.FAIL, 'failure', 'spam.ham',
+                           message='exception', extra_text='traceback')]
+    assert blocker.args == [expected]
 
-----------------------------------------------------------------------
-Ran 1 test in 0.000s
 
-OK
-"""
-    output = output11 if IS_PY311_OR_GREATER else output10
-    runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    res = runner.load_data(output)
-    assert len(res) == 1
-    assert res[0].category == Category.OK
-    assert res[0].status == 'ok'
-    assert res[0].name == 'test_foo.Bar.test1'
-    assert res[0].extra_text == []
-
-
-def test_unittestrunner_load_data_with_exception():
-    output10 = """test1 (test_foo.Bar) ... FAIL
-test2 (test_foo.Bar) ... ok
-
-======================================================================
-FAIL: test1 (test_foo.Bar)
-----------------------------------------------------------------------
-Traceback (most recent call last):
-  File "/somepath/test_foo.py", line 5, in test1
-    self.assertEqual(1, 2)
-AssertionError: 1 != 2
-
-----------------------------------------------------------------------
-Ran 2 tests in 0.012s
-
-FAILED (failures=1)
-"""
-    output11 = """test1 (test_foo.Bar.test1) ... FAIL
-test2 (test_foo.Bar.test2) ... ok
-
-======================================================================
-FAIL: test1 (test_foo.Bar.test1)
-----------------------------------------------------------------------
-Traceback (most recent call last):
-  File "/somepath/test_foo.py", line 5, in test1
-    self.assertEqual(1, 2)
-AssertionError: 1 != 2
-
-----------------------------------------------------------------------
-Ran 2 tests in 0.012s
-
-FAILED (failures=1)
-"""
-    output = output11 if IS_PY311_OR_GREATER else output10
-    runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    res = runner.load_data(output)
-    assert len(res) == 2
-
-    assert res[0].category == Category.FAIL
-    assert res[0].status == 'FAIL'
-    assert res[0].name == 'test_foo.Bar.test1'
-    assert res[0].extra_text[0].startswith('Traceback')
-    assert res[0].extra_text[-1].endswith('AssertionError: 1 != 2')
-
-    assert res[1].category == Category.OK
-    assert res[1].status == 'ok'
-    assert res[1].name == 'test_foo.Bar.test2'
-    assert res[1].extra_text == []
-
-
-def test_unittestrunner_load_data_with_comment():
-    output10 = """test1 (test_foo.Bar)
-comment ... ok
-test2 (test_foo.Bar) ... ok
-
-----------------------------------------------------------------------
-Ran 2 tests in 0.000s
-
-OK
-"""
-    output11 = """test1 (test_foo.Bar.test1)
-comment ... ok
-test2 (test_foo.Bar.test2) ... ok
-
-----------------------------------------------------------------------
-Ran 2 tests in 0.000s
-
-OK
-"""
-    output = output11 if IS_PY311_OR_GREATER else output10
-    runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    res = runner.load_data(output)
-    assert len(res) == 2
-
-    assert res[0].category == Category.OK
-    assert res[0].status == 'ok'
-    assert res[0].name == 'test_foo.Bar.test1'
-    assert res[0].extra_text == []
-
-    assert res[1].category == Category.OK
-    assert res[1].status == 'ok'
-    assert res[1].name == 'test_foo.Bar.test2'
-    assert res[1].extra_text == []
-
-
-def test_unittestrunner_load_data_with_fail_and_comment():
-    output10 = """test1 (test_foo.Bar)
-comment ... FAIL
-
-======================================================================
-FAIL: test1 (test_foo.Bar)
-comment
-----------------------------------------------------------------------
-Traceback (most recent call last):
-  File "/somepath/test_foo.py", line 30, in test1
-    self.assertEqual(1, 2)
-AssertionError: 1 != 2
-
-----------------------------------------------------------------------
-Ran 1 test in 0.000s
-
-FAILED (failures=1)
-"""
-    output11 = """test1 (test_foo.Bar.test1)
-comment ... FAIL
-
-======================================================================
-FAIL: test1 (test_foo.Bar.test1)
-comment
-----------------------------------------------------------------------
-Traceback (most recent call last):
-  File "/somepath/test_foo.py", line 30, in test1
-    self.assertEqual(1, 2)
-AssertionError: 1 != 2
-
-----------------------------------------------------------------------
-Ran 1 test in 0.000s
-
-FAILED (failures=1)
-"""
-    output = output11 if IS_PY311_OR_GREATER else output10
+def test_unittestrunner_process_output_with_adderror(qtbot):
+    """Test UnittestRunner.processOutput() with an `addError` event."""
     runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    res = runner.load_data(output)
-    assert len(res) == 1
+    output = [{'event': 'addError',
+               'id': 'spam.ham',
+               'reason': 'exception',
+               'err': 'traceback'}]
 
-    assert res[0].category == Category.FAIL
-    assert res[0].status == 'FAIL'
-    assert res[0].name == 'test_foo.Bar.test1'
-    assert res[0].extra_text[0].startswith('Traceback')
-    assert res[0].extra_text[-1].endswith('AssertionError: 1 != 2')
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
 
+    expected = [TestResult(Category.FAIL, 'error', 'spam.ham',
+                           message='exception', extra_text='traceback')]
+    assert blocker.args == [expected]
 
-def test_try_parse_header_with_ok():
+
+def test_unittestrunner_process_output_with_addskip(qtbot):
+    """Test UnittestRunner.processOutput() with an `addSkip` event."""
     runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    lines10 = ['test_isupper (testfoo.TestStringMethods) ... ok']
-    lines11 = ['test_isupper (testfoo.TestStringMethods.test_isupper) ... ok']
-    lines = lines11 if IS_PY311_OR_GREATER else lines10
-    res = runner.try_parse_result(lines, 0)
-    assert res == (1, 'testfoo.TestStringMethods.test_isupper', 'ok', '')
+    output = [{'event': 'addSkip',
+               'id': 'spam.ham',
+               'reason': 'skip reason'}]
 
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
 
-def test_try_parse_header_with_xfail():
-    runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    lines10 = ['test_isupper (testfoo.TestStringMethods) ... expected failure']
-    lines11 = ['test_isupper (testfoo.TestStringMethods.test_isupper) ... expected failure']
-    lines = lines11 if IS_PY311_OR_GREATER else lines10
-    res = runner.try_parse_result(lines, 0)
-    assert res == (1, 'testfoo.TestStringMethods.test_isupper',
-                   'expected failure', '')
+    expected = [TestResult(Category.SKIP, 'skip', 'spam.ham',
+                           message='skip reason')]
+    assert blocker.args == [expected]
 
 
-def test_try_parse_header_with_message():
+def test_unittestrunner_process_output_with_addexpectedfailure(qtbot):
+    """Test UnittestRunner.processOutput() with an `addExpectedFailure` event."""
     runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    lines10 = ["test_nothing (testfoo.Tests) ... skipped 'msg'"]
-    lines11 = ["test_nothing (testfoo.Tests.test_nothing) ... skipped 'msg'"]
-    lines = lines11 if IS_PY311_OR_GREATER else lines10
-    res = runner.try_parse_result(lines, 0)
-    assert res == (1, 'testfoo.Tests.test_nothing', 'skipped', 'msg')
+    output = [{'event': 'addExpectedFailure',
+               'id': 'spam.ham',
+               'reason': 'exception',
+               'err': 'traceback'}]
+
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
 
+    expected = [TestResult(Category.OK, 'expectedFailure', 'spam.ham',
+                           message='exception', extra_text='traceback')]
+    assert blocker.args == [expected]
 
-def test_try_parse_header_starting_with_digit():
+
+def test_unittestrunner_process_output_with_addunexpectedsuccess(qtbot):
+    """Test UnittestRunner.processOutput() with an `addUnexpectedSuccess` event."""
     runner = UnittestRunner(None)
-    runner.set_fullname_version()
-    lines10 = ['0est_isupper (testfoo.TestStringMethods) ... ok']
-    lines11 = ['0est_isupper (testfoo.TestStringMethods.0est_isupper) ... ok']
-    lines = lines11 if IS_PY311_OR_GREATER else lines10
-    res = runner.try_parse_result(lines, 0)
-    assert res is None
+    output = [{'event': 'addUnexpectedSuccess', 'id': 'spam.ham'}]
+
+    with qtbot.waitSignal(runner.sig_testresult) as blocker:
+        runner.process_output(output)
+
+    expected = [TestResult(Category.FAIL, 'unexpectedSuccess', 'spam.ham')]
+    assert blocker.args == [expected]