Add workaround to handle the testtool skip exception issue

stestr which is used by Tempest internally to run the test switch
the customize test runner(which use stdlib unittest) for >=py3.5
else testtools.run.- mtreinish/stestr#265

These two test runner are not compatible due to skip exception
handling(due to unittest2). testtools.run treat unittestt.SkipTest
as error and stdlib unittest treat unittest2.case.SkipTest raised
by testtools.TestCase.skipException.

testtool issue: testing-cabal/testtools#272

testtool is not so active now a days and fix is also not decided,
let's add a workaround in Tempest to make it work for both test runner
based on python version same as used by stestr

Tempest should work perfectly with stestr<2.5.0 also which is
also handled in this workaround.

Change-Id: Ie9c013d4d6851d4deef57c1e4c254a9a34374e5a

tempest/test.py

@@ -20,6 +20,7 @@
 import debtcollector.moves
 import fixtures
 from oslo_log import log as logging
+import pkg_resources
 import six
 import testtools
 
@@ -77,6 +78,10 @@ def validate_tearDownClass():
 atexit.register(validate_tearDownClass)
 
 
+class DummyException(Exception):
+    pass
+
+
 class BaseTestCase(testtools.testcase.WithAttributes,
                    testtools.TestCase):
     """The test base class defines Tempest framework for class level fixtures.
@@ -139,6 +144,26 @@ def _reset_class(cls):
         # Stack of (name, callable) to be invoked in reverse order at teardown
         cls._teardowns = []
 
+    @classmethod
+    def handle_skip_exception(cls):
+        try:
+            stestr_version = pkg_resources.parse_version(
+                pkg_resources.get_distribution("stestr").version)
+            stestr_min = pkg_resources.parse_version('2.5.0')
+            new_stestr = (stestr_version >= stestr_min)
+            import unittest
+            import unittest2
+            if sys.version_info >= (3, 5) and new_stestr:
+                exc = unittest2.case.SkipTest
+                exc_to_raise = unittest.case.SkipTest
+            else:
+                exc = unittest.case.SkipTest
+                exc_to_raise = unittest2.case.SkipTest
+        except Exception:
+            exc = DummyException
+            exc_to_raise = DummyException
+        return exc, exc_to_raise
+
     @classmethod
     def setUpClass(cls):
         cls.__setupclass_called = True
@@ -148,11 +173,24 @@ def setUpClass(cls):
         if hasattr(super(BaseTestCase, cls), 'setUpClass'):
             super(BaseTestCase, cls).setUpClass()
         # All the configuration checks that may generate a skip
-        cls.skip_checks()
-        if not cls.__skip_checks_called:
-            raise RuntimeError("skip_checks for %s did not call the super's "
-                               "skip_checks" % cls.__name__)
+        # TODO(gmann): cls.handle_skip_exception is really workaround for
+        # testtools bug- https://github.com/testing-cabal/testtools/issues/272
+        # stestr which is used by Tempest internally to run the test switch
+        # the customize test runner(which use stdlib unittest) for >=py3.5
+        # else testtools.run.- https://github.com/mtreinish/stestr/pull/265
+        # These two test runner are not compatible due to skip exception
+        # handling(due to unittest2). testtools.run treat unittestt.SkipTest
+        # as error and stdlib unittest treat unittest2.case.SkipTest raised
+        # by testtools.TestCase.skipException.
+        # The below workaround can be removed once testtools fix issue# 272.
         try:
+            exc, exc_to_raise = cls.handle_skip_exception()
+            cls.skip_checks()
+
+            if not cls.__skip_checks_called:
+                raise RuntimeError(
+                    "skip_checks for %s did not call the super's "
+                    "skip_checks" % cls.__name__)
             # Allocation of all required credentials and client managers
             cls._teardowns.append(('credentials', cls.clear_credentials))
             cls.setup_credentials()
@@ -164,6 +202,8 @@ def setUpClass(cls):
             # Additional class-wide test resources
             cls._teardowns.append(('resources', cls.resource_cleanup))
             cls.resource_setup()
+        except exc as e:
+            raise exc_to_raise(e.args)
         except Exception:
             etype, value, trace = sys.exc_info()
             LOG.info("%s raised in %s.setUpClass. Invoking tearDownClass.",

tempest/tests/test_test.py

@@ -531,8 +531,8 @@ def test_no_error_flow(self):
     def test_skip_only(self):
         # If a skip condition is hit in the test, no credentials or resource
         # is provisioned / cleaned-up
-        self.mocks['skip_checks'].side_effect = (
-            testtools.testcase.TestSkipped())
+        exc, _ = test.BaseTestCase.handle_skip_exception()
+        self.mocks['skip_checks'].side_effect = (exc)
         suite = unittest.TestSuite((self.test,))
         log = []
         result = LoggingTestResult(log)