Issue related to #1861 and differences in "unittest" between Python 3.10 and 3.11.

[coseto6125]

self.save_teardown_screenshot() -> self.__check_scope() -> unittest.has_exception = False

The call to self.save_teardown_screenshot() is followed by a call to self.__check_scope(),
which in turn resets unittest.has_exception to False.
As a result, the subsequent call to self.has_exception() will also return False, causing the test to print passed again.

same code here.

from seleniumbase import BaseCase

BaseCase.main(__name__, __file__)


class BaseTestCase(BaseCase):

    def setUp(self):
        super().setUp()
        # <<< Run custom setUp() code for tests AFTER the super().setUp() >>>

    def tearDown(self):
        self.save_teardown_screenshot()  # If test fails, or if "--screenshot"
        if self.has_exception():
            # <<< Run custom code if the test failed. >>>
            print("fail")
        else:
            # <<< Run custom code if the test passed. >>>
            print("passed")
        # (Wrap unreliable tearDown() code in a try/except block.)
        # <<< Run custom tearDown() code BEFORE the super().tearDown() >>>
        super().tearDown()


class TestFailing(BaseTestCase):

    def test_find_army_of_robots_on_xkcd_desert_island(self):
        self.open("https://xkcd.com/731/")
        print("\n(This test should fail)")
        self.assert_element("div#ARMY_OF_ROBOTS", timeout=1)

I noticed that in version 3.11 of unittest,
the testPartExecutor method of the _Outcome class handles raising exceptions differently than in previous versions.
Instead of being aggregated into self._outcome, they are aggregated into a self.result object that seems to be aggregated only after the test has completed.
This results in self.has_exception() being unable to correctly determine if an exception occurred.

I tried adding a run method to BaseCase to override the original run function,
mainly to cover the testPartExecutor method called in run.

  def run(self, result=None): #copy from python 3.9.6
        if result is None:
            result = self.defaultTestResult()
            startTestRun = getattr(result, 'startTestRun', None)
            stopTestRun = getattr(result, 'stopTestRun', None)
            if startTestRun is not None:
                startTestRun()
        else:
            stopTestRun = None

        result.startTest(self)
        try:
            testMethod = getattr(self, self._testMethodName)
            if (getattr(self.__class__, "__unittest_skip__", False) or
                getattr(testMethod, "__unittest_skip__", False)):
                # If the class or method was skipped.
                skip_why = (getattr(self.__class__, '__unittest_skip_why__', '')
                            or getattr(testMethod, '__unittest_skip_why__', ''))
                _addSkip(result, self, skip_why)
                return result

            expecting_failure = (
                getattr(self, "__unittest_expecting_failure__", False) or
                getattr(testMethod, "__unittest_expecting_failure__", False)
            )
            outcome = Outcome(result)
            try:
                self._outcome = outcome

                with outcome.testPartExecutor(self):
                    self._callSetUp()
                if outcome.success:
                    outcome.expecting_failure = expecting_failure
                    with outcome.testPartExecutor(self):
                        self._callTestMethod(testMethod)
                    outcome.expecting_failure = False
                    with outcome.testPartExecutor(self):
                        self._callTearDown()
                self.doCleanups()

                if outcome.success:
                    if expecting_failure:
                        if outcome.expectedFailure:
                            self._addExpectedFailure(result, outcome.expectedFailure)
                        else:
                            self._addUnexpectedSuccess(result)
                    else:
                        result.addSuccess(self)
                return result
            finally:
                # explicitly break reference cycle:
                # outcome.expectedFailure -> frame -> outcome -> outcome.expectedFailure
                outcome.expectedFailure = None
                outcome = None

                # clear the outcome, no more needed
                self._outcome = None

class Outcome(object): #copy from python 3.9.6
    def __init__(self, result=None):
        self.expecting_failure = False
        self.result = result
        self.result_supports_subtests = hasattr(result, "addSubTest")
        self.success = True
        self.skipped = []
        self.expectedFailure = None
        self.errors = []
    @contextlib.contextmanager
    def testPartExecutor(self, test_case, isTest=False):
        old_success = self.success
        self.success = True
        try:
            yield
        except KeyboardInterrupt:
            raise
        except SkipTest as e:
            self.success = False
            self.skipped.append((test_case, str(e)))
        except _ShouldStop:
            pass
        except:
            exc_info = sys.exc_info()
            if self.expecting_failure:
                self.expectedFailure = exc_info
            else:
                self.success = False
                self.errors.append((test_case, exc_info))
            # explicitly break a reference cycle:
            # exc_info -> frame -> exc_info
            exc_info = None
        else:
            if self.result_supports_subtests and self.success:
                self.errors.append((test_case, None))
        finally:
            self.success = self.success and old_success

by the way,
I think that in order to avoid potential changes in future versions, in addition to using monkey patching methods like this,
it might be worth considering extracting the older version of unittest and including it in the Selenium base package.
This would help to prevent any further changes from affecting the overall package,
especially since BaseCase mainly inherits from unittest.

[mdmintz]

Sounds good. I'll monkey-patch SeleniumBase to include the older unittest code for Python 3.11 or newer. I'll see how fast I can make this change...

[mdmintz]

This was resolved by 4.14.6 - https://github.com/seleniumbase/SeleniumBase/releases/tag/v4.14.6

[mdmintz]

And thank you, @coseto6125, for the detailed bug report. That made it much easier for me to fix this issue because that helped me understand the exact changes that needed to be made.