Merge pull request #132 from gregoil/add_success_message

Add success message to tests

docs/intro/more_on_testing.rst

@@ -148,34 +148,70 @@ Now, let's run the test:
     OK
 
 
-Assert vs Expect
-================
+Test event methods
+==================
 
-In the test method you can use the assert<X> methods to perform the testing,
-but for cases where you don't want the action to stop the test, you can use ``expect``.
+Test result events you can use in Rotest:
 
-``expect`` only registers failures but stays in the same scope,
-allowing for more testing actions in the same single test. E.g.
+* `self.fail(<message>)`, `self.skip(<message>)` as in ``unittest``.
 
-.. code-block:: python
+* All failure events using `assert<X>`, as in ``unittest``.
 
-    from rotest.core import TestCase
+* `expect<X>` methods (a new concept) - for cases where you want to fail the
+  test but don't want the action to break the test flow.
 
-    from resources.calculator import Calculator
+  ``expect`` only registers the failures (if there are any) but stays in the same
+  scope, allowing for more testing actions in the same single test. E.g.
 
+  .. code-block:: python
 
-    class AddTest(TestCase):
-        calc = Calculator()
+      from rotest.core import TestCase
+
+      from resources.calculator import Calculator
+
+
+      class AddTest(TestCase):
+            calc = Calculator()
+
+          def test_add(self):
+              self.expectEqual(self.calc.calculate("1 + 1"), 2)
+              self.expectEqual(self.calc.calculate("1 + 2"), 2)
+              self.expectEqual(self.calc.calculate("1 + 3"), 2)
+
+  In the above example ``AddTest`` will have 2 failures to the same run (3!=2 and 4!=2).
+
+  It is recommended to use ``expect`` to test different side-effects of the same scenario,
+  like different side effects of the same action, but you can use it any way you please.
+
+  There is an ``expect`` method equivalent for every ``assert`` method, e.g. ``expectEqual`` and ``expectIsNone``.
+
+* Success events (a new concept) - When you want to register information about the
+  test, like numeric results of actions or time measurement of actions.
+
+  The success information will be registered into the test's metadata, like any
+  other failure, error, or skip message, and will be visible in the DB, excel, etc.
+
+  .. code-block:: python
+
+      from rotest.core import TestCase
+
+      from resources.calculator import Calculator
 
-        def test_add(self):
-            self.expectEqual(self.calc.calculate("1 + 1"), 2)
-            self.expectEqual(self.calc.calculate("1 + 2"), 2)
-            self.expectEqual(self.calc.calculate("1 + 3"), 2)
 
+      class AddTest(TestCase):
+            calc = Calculator()
 
-In the above example the ``AddTest`` will have 2 failures to the same run (3!=2 and 4!=2).
+          def test_add(self):
 
-It is recommended to use ``expect`` to test different side-effects of the same scenario,
-like different side effects of the same action, but you can use it any way you please.
+              self.success("One way to register success")
+              # Or
+              self.addSuccess("Another way to register success")
 
-There is an ``expect`` method equivalent for every ``assert`` method, e.g. ``expectEqual`` and ``expectIsNone``.
+              value = self.calc.calculate("1 + 1")
+              self.expectEqual(value, 3,
+                               msg="Expected value 3, got %r" % value,
+                               success_msg="Value is %r, as expected" % value)
+              # Or
+              self.assertEqual(value, 3,
+                               msg="Expected value 3, got %r" % value,
+                               success_msg="Value is %r, as expected" % value)

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 
-__version__ = "6.1.0"
+__version__ = "6.2.0"
 
 result_handlers = [
     "db = rotest.core.result.handlers.db_handler:DBHandler",

src/rotest/core/abstract_test.py

@@ -360,103 +360,6 @@ def _wrap_assert(self, assert_method, *args, **kwargs):
         except AssertionError as err:
             self.expect(False, str(err))
 
-    def expectFalse(self, expr, msg=None):
-        self._wrap_assert(self.assertFalse, expr, msg)
-
-    def expectTrue(self, expr, msg=None):
-        self._wrap_assert(self.assertTrue, expr, msg)
-
-    def expectEqual(self, first, second, msg=None):
-        self._wrap_assert(self.assertEqual, first, second, msg)
-
-    def expectNotEqual(self, first, second, msg=None):
-        self._wrap_assert(self.assertNotEqual, first, second, msg)
-
-    def expectAlmostEqual(self, first, second, places=None,
-                          msg=None, delta=None):
-
-        self._wrap_assert(self.assertAlmostEqual, first, second, places,
-                          msg, delta)
-
-    def expectNotAlmostEqual(self, first, second, places=None,
-                             msg=None, delta=None):
-
-        self._wrap_assert(self.assertNotAlmostEqual, first, second, places,
-                          msg, delta)
-
-    expectEquals = expectEqual
-    expectNotEquals = expectNotEqual
-    expectAlmostEquals = expectAlmostEqual
-    expectNotAlmostEquals = expectNotAlmostEqual
-
-    def expectSequenceEqual(self, seq1, seq2, msg=None, seq_type=None):
-        self._wrap_assert(self.assertSequenceEqual, seq1, seq2, msg, seq_type)
-
-    def expectListEqual(self, list1, list2, msg=None):
-        self._wrap_assert(self.assertListEqual, list1, list2, msg)
-
-    def expectTupleEqual(self, tuple1, tuple2, msg=None):
-        self._wrap_assert(self.assertTupleEqual, tuple1, tuple2, msg)
-
-    def expectSetEqual(self, set1, set2, msg=None):
-        self._wrap_assert(self.assertSetEqual, set1, set2, msg)
-
-    def expectIn(self, member, container, msg=None):
-        self._wrap_assert(self.assertIn, member, container, msg)
-
-    def expectNotIn(self, member, container, msg=None):
-        self._wrap_assert(self.assertNotIn, member, container, msg)
-
-    def expectIs(self, expr1, expr2, msg=None):
-        self._wrap_assert(self.assertIs, expr1, expr2, msg)
-
-    def expectIsNot(self, expr1, expr2, msg=None):
-        self._wrap_assert(self.assertIsNot, expr1, expr2, msg)
-
-    def expectDictEqual(self, set1, set2, msg=None):
-        self._wrap_assert(self.assertDictEqual, set1, set2, msg)
-
-    def expectDictContainsSubset(self, expected, actual, msg=None):
-        self._wrap_assert(self.assertDictContainsSubset, expected, actual, msg)
-
-    def expectItemsEqual(self, expected_seq, actual_seq, msg=None):
-        self._wrap_assert(self.assertItemsEqual, expected_seq, actual_seq, msg)
-
-    def expectMultiLineEqual(self, first, second, msg=None):
-        self._wrap_assert(self.assertMultiLineEqual, first, second, msg)
-
-    def expectLess(self, a, b, msg=None):
-        self._wrap_assert(self.assertLess, a, b, msg)
-
-    def expectLessEqual(self, a, b, msg=None):
-        self._wrap_assert(self.assertLessEqual, a, b, msg)
-
-    def expectGreater(self, a, b, msg=None):
-        self._wrap_assert(self.assertGreater, a, b, msg)
-
-    def expectGreaterEqual(self, a, b, msg=None):
-        self._wrap_assert(self.assertGreaterEqual, a, b, msg)
-
-    def expectIsNone(self, obj, msg=None):
-        self._wrap_assert(self.assertIsNone, obj, msg)
-
-    def expectIsNotNone(self, obj, msg=None):
-        self._wrap_assert(self.assertIsNotNone, obj, msg)
-
-    def expectIsInstance(self, obj, msg=None):
-        self._wrap_assert(self.assertIsInstance, obj, msg)
-
-    def expectNotIsInstance(self, obj, msg=None):
-        self._wrap_assert(self.assertNotIsInstance, obj, msg)
-
-    def expectRegexpMatches(self, text, expected_regexp, msg=None):
-        self._wrap_assert(self.assertRegexpMatches, text,
-                          expected_regexp, msg)
-
-    def expectNotRegexpMatches(self, text, unexpected_regexp, msg=None):
-        self._wrap_assert(self.assertNotRegexpMatches, text,
-                          unexpected_regexp, msg)
-
     class _ExpectRaisesContext(object):
         def __init__(self, assert_context, wrap_assert):
             self.assert_context = assert_context
@@ -495,3 +398,45 @@ def expectRaisesRegexp(self, expected_exception, expected_regexp,
 
         self._wrap_assert(self.assertRaisesRegexp, expected_exception,
                           expected_regexp, callable_obj, *args, **kwargs)
+
+    def addSuccess(self, msg):
+        """Register a success message to the test result.
+
+        Args:
+            msg (str): success message to add to the result.
+        """
+        self.result.addSuccess(self, msg)
+
+    # Shortcuts
+    success = addSuccess
+    skip = unittest.TestCase.skipTest
+
+
+def create_expect_method(method_name):
+    original_assert = getattr(unittest.TestCase, method_name)
+    @wraps(original_assert)
+    def extended_assert(self, *args, **kwargs):
+        success_msg = kwargs.pop("success_msg", None)
+        retval = original_assert(self, *args, **kwargs)
+        if success_msg is not None:
+            self.success(success_msg)
+        return retval
+
+    setattr(AbstractTest, method_name, extended_assert)
+
+    def expect_func(self, *args, **kwargs):
+        return self._wrap_assert(getattr(self, method_name), *args, **kwargs)
+
+    expect_func.__doc__ = """Like {} but doesn't break workflow.""".format(
+                                                                method_name)
+
+    setattr(AbstractTest, method_name.replace("assert", "expect"),
+            expect_func)
+
+
+# Create an 'expect' method for every 'assert' method in unittest.TestCase
+for attr_name in unittest.TestCase.__dict__:
+    if attr_name.startswith("assert") and \
+            "Raises" not in attr_name and "_" not in attr_name:
+
+        create_expect_method(attr_name)

src/rotest/core/models/case_data.py

@@ -47,7 +47,7 @@ class CaseData(GeneralData):
     TB_SEPARATOR = 80 * '-' + '\n'
     _RUNTIME_ORDER = '-start_time'
 
-    RESULT_CHOICES = {TestOutcome.SUCCESS: 'OK',
+    RESULT_CHOICES = {TestOutcome.SUCCESS: 'Success',
                       TestOutcome.ERROR: 'Error',
                       TestOutcome.FAILED: 'Failed',
                       TestOutcome.SKIPPED: 'Skipped',
@@ -131,8 +131,9 @@ def update_result(self, result_type, details=None):
 
             self.exception_type = result_type
 
-        if result_type not in (TestOutcome.SUCCESS,
-                               TestOutcome.UNEXPECTED_SUCCESS):
+        if details is not None:
+            details = "{}: {}".format(self.RESULT_CHOICES[result_type].upper(),
+                                      details)
 
             if len(self.traceback) > 0:
                 details = self.TB_SEPARATOR.join([self.traceback, details])

src/rotest/core/result/handlers/abstract_handler.py

@@ -97,11 +97,12 @@ def stop_test_run(self):
         """Called once after all tests are executed."""
         pass
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Called when a test has completed successfully.
 
         Args:
             test (rotest.core.abstract_test.AbstractTest): test item instance.
+            msg (str): success message.
         """
         pass
 

src/rotest/core/result/handlers/db_handler.py

@@ -140,11 +140,12 @@ def stop_composite(self, test):
         """
         test.data.save()
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Save the test data result as success.
 
         Args:
             test (object): test item instance.
+            msg (str): success message.
         """
         test.data.save()
 

src/rotest/core/result/handlers/excel_handler.py

@@ -207,11 +207,12 @@ def update_resources(self, test):
         self._write_to_cell(self.test_to_row[test.identifier], self.RESOURCES,
                             self.DEFAULT_CELL_STYLE, resources)
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Update the test Excel entry's result to success.
 
         Args:
             test (object): test item instance.
+            msg (str): success message.
         """
         self._write_test_result(test)
         self.workbook.save(self.output_file_path)

src/rotest/core/result/handlers/remote_db_handler.py

@@ -95,13 +95,14 @@ def stop_composite(self, test):
         """
         self.client.stop_composite(test)
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Save the remote test data result as success.
 
         Args:
             test (object): test item instance.
+            msg (str): success message.
         """
-        self.client.add_result(test, TestOutcome.SUCCESS)
+        self.client.add_result(test, TestOutcome.SUCCESS, msg)
 
     def add_skip(self, test, reason):
         """Save the remote test data result as skip.

src/rotest/core/result/handlers/stream/dots_handler.py

@@ -35,11 +35,12 @@ class DotsHandler(BaseStreamHandler):
     NAME = 'dots'
 
     @ignore_subtests
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Write the test success to the stream.
 
         Args:
             test (rotest.core.case.TestCase): test item instance.
+            msg (str): success message.
         """
         self.stream.write('.', GREEN)
 

src/rotest/core/result/handlers/stream/log_handler.py

@@ -61,7 +61,7 @@ class PrettyHandler(LogStreamHandler):
     def start_test(self, test):
         self.stream.writeln(str(Pretty(test, TestResult.started)))
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         self.stream.writeln(str(Pretty(test, TestResult.success)))
 
     def add_skip(self, test, reason):

src/rotest/core/result/handlers/stream/stream_handler.py

@@ -54,13 +54,16 @@ def stop_test_run(self):
         """Write the test run end to the stream."""
         self.stream.writeln('Tests Run Finished', None, BOLD)
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Write the test success to the stream.
 
         Args:
             test (TestCase): test item instance.
+            msg (str): success message.
         """
         self.stream.writeln('Success: %s' % test, GREEN)
+        if msg is not None:
+            self.write_details(msg, color=GREEN)
 
     def add_skip(self, test, reason):
         """Write the test skip to the stream.

src/rotest/core/result/handlers/stream/tree_handler.py

@@ -46,11 +46,12 @@ def start_composite(self, test):
         """
         self.start_test(test)
 
-    def add_success(self, test):
+    def add_success(self, test, msg):
         """Write the test success to the stream.
 
         Args:
             test (TestCase): test item instance.
+            msg (str): success message.
         """
         if test.IS_COMPLEX:
             indentation = test.parents_count * self.INDENTATION