diff --git a/requirements.txt b/requirements.txt index b1ef8657..cc5d282e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ jsonschema==2.6.0 lazy-object-proxy==1.3.1 mccabe==0.6.1 mock==2.0.0 -more-itertools==4.3.0 +more-itertools==5.0.0 pathlib2==2.3.3 pbr==5.1.1 pexpect==4.6.0 @@ -39,12 +39,12 @@ psutil==5.4.8 ptyprocess==0.6.0 py==1.7.0 pycodestyle==2.4.0 -pyfakefs==3.5.4 +pyfakefs==3.5.5 pyflakes==2.0.0 Pygments==2.3.1 -pylint==1.9.3 -pytest==4.0.2 -pytest-cov==2.6.0 +pylint==1.9.4 +pytest==4.1.0 +pytest-cov==2.6.1 pytest-django==3.4.4 PyYAML==3.13 requests==2.21.0 diff --git a/setup.py b/setup.py index dde5fea4..5c063ab7 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages -__version__ = "5.4.1" +__version__ = "5.4.2" result_handlers = [ "db = rotest.core.result.handlers.db_handler:DBHandler", diff --git a/src/rotest/core/abstract_test.py b/src/rotest/core/abstract_test.py index 9744860d..fca092c0 100644 --- a/src/rotest/core/abstract_test.py +++ b/src/rotest/core/abstract_test.py @@ -19,9 +19,11 @@ from future.builtins import next, str, object from future.utils import iteritems, itervalues +from rotest.core.result.result import Result from rotest.common.utils import get_class_fields from rotest.core.models.case_data import TestOutcome from rotest.management.base_resource import BaseResource +from rotest.common.log import get_test_logger, get_tree_path from rotest.management.client.manager import ResourceRequest from rotest.management.client.manager import ClientResourceManager @@ -64,9 +66,9 @@ class AbstractTest(unittest.TestCase): STATE_DIR_NAME = "state" - def __init__(self, indexer=count(), methodName='runTest', save_state=True, - force_initialize=False, config=None, parent=None, - enable_debug=True, resource_manager=None, skip_init=False): + def __init__(self, methodName='test_method', indexer=count(), parent=None, + save_state=True, force_initialize=False, config=None, + enable_debug=False, resource_manager=None, skip_init=False): if enable_debug: for method_name in (methodName, self.SETUP_METHOD_NAME, @@ -80,6 +82,8 @@ def __init__(self, indexer=count(), methodName='runTest', save_state=True, super(AbstractTest, self).__init__(methodName) self.result = None + self.logger = None + self.is_main = True self.config = config self.parent = parent self.skip_init = skip_init @@ -201,7 +205,7 @@ def request_resources(self, resources_to_request, use_previous=False): for resource in itervalues(requested_resources): resource.override_logger(self.logger) - if self.result is not None: + if isinstance(self.result, Result): self.result.updateResources(self) def release_resources(self, resources=None, dirty=False, @@ -248,6 +252,12 @@ def _get_parents_count(self): return self.parent.parents_count + 1 + def create_logger(self): + """Create logger instance for the test.""" + if self.logger is None: + self.logger = get_test_logger(get_tree_path(self), self.work_dir) + self.logger.info("Test %r has started running", self.data) + def start(self): """Update the data that the test started.""" self.data.start() @@ -262,12 +272,11 @@ def end(self, test_outcome, details=None): """ self.data.update_result(test_outcome, details) - def _decorate_teardown(self, teardown_method, result): + def _decorate_teardown(self, teardown_method): """Decorate the tearDown method to handle resource release. Args: teardown_method (function): the original tearDown method. - result (rotest.core.result.result.Result): test result information. Returns: function. the wrapped tearDown method. @@ -280,12 +289,14 @@ def teardown_method_wrapper(*args, **kwargs): * Releases the test resources. * Closes the client if needed """ - self.result.startTeardown(self) + if isinstance(self.result, Result): + self.result.startTeardown(self) + try: teardown_method(*args, **kwargs) except Exception: - result.addError(self, sys.exc_info()) + self.result.addError(self, sys.exc_info()) finally: self.store_state() diff --git a/src/rotest/core/block.py b/src/rotest/core/block.py index a8878113..f2a8e74b 100755 --- a/src/rotest/core/block.py +++ b/src/rotest/core/block.py @@ -73,10 +73,11 @@ class TestBlock(AbstractFlowComponent): IS_COMPLEX (bool): if this test is complex (may contain sub-tests). """ IS_COMPLEX = False + __test__ = False def __init__(self, indexer=count(), base_work_dir=ROTEST_WORK_DIR, save_state=True, force_initialize=False, config=None, - parent=None, run_data=None, enable_debug=True, + parent=None, run_data=None, enable_debug=False, resource_manager=None, skip_init=False, is_main=True): super(TestBlock, self).__init__(parent=parent, @@ -131,8 +132,8 @@ def _share_outputs(self): outputs_dict = {} for output_name in self.get_outputs(): if output_name not in self.__dict__: - self.logger.warn("Block %r didn't create output %r", - self.data.name, output_name) + self.logger.warning("Block %r didn't create output %r", + self.data.name, output_name) outputs_dict[output_name] = getattr(self, output_name) diff --git a/src/rotest/core/case.py b/src/rotest/core/case.py index 8ac57037..23408c8b 100644 --- a/src/rotest/core/case.py +++ b/src/rotest/core/case.py @@ -9,6 +9,7 @@ from rotest.common import core_log from rotest.common.utils import get_work_dir +from rotest.core.result.result import Result from rotest.common.config import ROTEST_WORK_DIR from rotest.core.models.case_data import CaseData from rotest.core.abstract_test import AbstractTest, request @@ -54,19 +55,20 @@ class TestCase(AbstractTest): IS_COMPLEX = False test_methods_names = None - def __init__(self, indexer=count(), methodName='runTest', + def __init__(self, methodName='test_method', parent=None, indexer=count(), base_work_dir=ROTEST_WORK_DIR, save_state=True, - force_initialize=False, config=None, parent=None, - run_data=None, enable_debug=True, resource_manager=None, - skip_init=False): - - super(TestCase, self).__init__(indexer, methodName, save_state, - force_initialize, config, parent, - enable_debug, resource_manager, - skip_init) - - self.skip_reason = None - self.skip_determined = False + force_initialize=False, config=None, run_data=None, + enable_debug=False, resource_manager=None, skip_init=False): + + super(TestCase, self).__init__(methodName=methodName, + parent=parent, + indexer=indexer, + save_state=save_state, + force_initialize=force_initialize, + config=config, + enable_debug=enable_debug, + resource_manager=resource_manager, + skip_init=skip_init) name = self.get_name(methodName) core_log.debug("Initializing %r test-case", name) @@ -120,16 +122,21 @@ def setup_method_wrapper(*args, **kwargs): * Executes the original setUp method. * Upon exception, finalizes the resources. """ - skip_reason = self.result.shouldSkip(self) - if skip_reason is not None: - self.skipTest(skip_reason) + self.override_resource_loggers() + + if isinstance(self.result, Result): + skip_reason = self.result.shouldSkip(self) + + if skip_reason is not None: + self.skipTest(skip_reason) self.request_resources(self.get_resource_requests(), use_previous=True) try: setup_method(*args, **kwargs) - self.result.setupFinished(self) + if isinstance(self.result, Result): + self.result.setupFinished(self) except Exception: self.release_resources(dirty=True) @@ -151,6 +158,7 @@ def run(self, result=None): # method signature, but the Rotest test case does not support it. self.assertIsNotNone(result, 'TestCase must run inside a TestSuite') self.result = result + self.create_logger() # === Decorate the setUp, test and tearDown methods. === setup_method = getattr(self, self.SETUP_METHOD_NAME) @@ -159,6 +167,6 @@ def run(self, result=None): teardown_method = getattr(self, self.TEARDOWN_METHOD_NAME) setattr(self, self.TEARDOWN_METHOD_NAME, - self._decorate_teardown(teardown_method, result)) + self._decorate_teardown(teardown_method)) super(TestCase, self).run(result) diff --git a/src/rotest/core/flow.py b/src/rotest/core/flow.py index cade9e81..f848d3ef 100644 --- a/src/rotest/core/flow.py +++ b/src/rotest/core/flow.py @@ -83,8 +83,8 @@ class TestFlow(AbstractFlowComponent): TEST_METHOD_NAME = "test_run_blocks" - def __init__(self, base_work_dir=ROTEST_WORK_DIR, save_state=True, - force_initialize=False, config=None, indexer=count(), + def __init__(self, indexer=count(), base_work_dir=ROTEST_WORK_DIR, + save_state=True, force_initialize=False, config=None, parent=None, run_data=None, enable_debug=False, is_main=True, skip_init=False, resource_manager=None): @@ -135,6 +135,12 @@ def __iter__(self): def addTest(self, test_item): self._tests.append(test_item) + def create_logger(self): + """Create logger instances for the test and its components.""" + super(TestFlow, self).create_logger() + for block in self._tests: + block.create_logger() + def validate_inputs(self, extra_inputs=[]): """Validate that all the required inputs of the blocks were passed. diff --git a/src/rotest/core/flow_component.py b/src/rotest/core/flow_component.py index 63e98548..e90620db 100644 --- a/src/rotest/core/flow_component.py +++ b/src/rotest/core/flow_component.py @@ -3,6 +3,7 @@ # pylint: disable=too-many-arguments,too-many-locals,broad-except # pylint: disable=dangerous-default-value,access-member-before-definition # pylint: disable=bare-except,protected-access,too-many-instance-attributes +# pylint: disable=too-many-branches from __future__ import absolute_import, print_function import unittest @@ -14,6 +15,7 @@ from rotest.common import core_log from rotest.common.utils import get_work_dir +from rotest.core.result.result import Result from rotest.common.config import ROTEST_WORK_DIR from rotest.core.abstract_test import AbstractTest from rotest.management.common.errors import ServerError @@ -118,14 +120,20 @@ class AbstractFlowComponent(AbstractTest): def __init__(self, indexer=count(), base_work_dir=ROTEST_WORK_DIR, save_state=True, force_initialize=False, config=None, - parent=None, run_data=None, enable_debug=True, + parent=None, run_data=None, enable_debug=False, resource_manager=None, skip_init=False, is_main=True): test_method_name = self.get_test_method_name() - super(AbstractFlowComponent, self).__init__(indexer, test_method_name, - save_state, force_initialize, config, - parent, enable_debug, resource_manager, - skip_init) + super(AbstractFlowComponent, self).__init__( + methodName=test_method_name, + parent=parent, + indexer=indexer, + save_state=save_state, + force_initialize=force_initialize, + config=config, + enable_debug=enable_debug, + resource_manager=resource_manager, + skip_init=skip_init) self._pipes = {} self.is_main = is_main @@ -219,11 +227,15 @@ def setup_method_wrapper(*args, **kwargs): * Executes the original setUp method. * Upon exception, finalizes the resources. """ + self.override_resource_loggers() + if self.is_main: - skip_reason = self.result.shouldSkip(self) - if skip_reason is not None: - self.skip_sub_components(skip_reason) - self.skipTest(skip_reason) + if isinstance(self.result, Result): + skip_reason = self.result.shouldSkip(self) + + if skip_reason is not None: + self.skip_sub_components(skip_reason) + self.skipTest(skip_reason) else: if self.mode in (MODE_CRITICAL, MODE_OPTIONAL): @@ -263,7 +275,8 @@ def setup_method_wrapper(*args, **kwargs): self.validate_inputs() setup_method(*args, **kwargs) - self.result.setupFinished(self) + if isinstance(self.result, Result): + self.result.setupFinished(self) except Exception: self.release_resources(self.locked_resources, dirty=True) @@ -290,6 +303,7 @@ def run(self, result=None): result (rotest.core.result.result.Result): test result information. """ self.result = result + self.create_logger() # === Decorate the setUp and tearDown methods === setup_method = getattr(self, self.SETUP_METHOD_NAME) @@ -298,7 +312,7 @@ def run(self, result=None): teardown_method = getattr(self, self.TEARDOWN_METHOD_NAME) setattr(self, self.TEARDOWN_METHOD_NAME, - self._decorate_teardown(teardown_method, result)) + self._decorate_teardown(teardown_method)) try: super(AbstractFlowComponent, self).run(result) diff --git a/src/rotest/core/result/result.py b/src/rotest/core/result/result.py index 98968ec7..8a17e976 100644 --- a/src/rotest/core/result/result.py +++ b/src/rotest/core/result/result.py @@ -7,8 +7,6 @@ from rotest.common import core_log from rotest.core.models.case_data import TestOutcome -from rotest.core.flow_component import AbstractFlowComponent -from rotest.common.log import get_test_logger, get_tree_path def get_result_handlers(): @@ -62,12 +60,9 @@ def startTest(self, test): Args: test (object): test item instance. """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).startTest(test) - test.logger = get_test_logger(get_tree_path(test), test.work_dir) - test.logger.info("Test %r has started running", test.data) - test.override_resource_loggers() test.start() for result_handler in self.result_handlers: @@ -130,7 +125,7 @@ def stopTest(self, test): Args: test (object): test item instance. """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).stopTest(test) test.logger.debug("Test %r has stopped running", test.data) @@ -206,7 +201,7 @@ def addSuccess(self, test): if test.data.exception_type is not None: return - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addSuccess(test) test.logger.info("Test %r ended successfully", test.data) @@ -222,7 +217,7 @@ def addSkip(self, test, reason): test (object): test item instance. reason (str): skip reason description. """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addSkip(test, reason) test.logger.warning("Test %r skipped, reason %r", test.data, reason) @@ -238,7 +233,7 @@ def addFailure(self, test, err): test (object): test item instance. err (tuple): tuple of values as returned by sys.exc_info(). """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addFailure(test, err) exception_string = self._exc_info_to_string(err, test) @@ -256,7 +251,7 @@ def addError(self, test, err): test (object): test item instance. err (tuple): tuple of values as returned by sys.exc_info(). """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addError(test, err) exception_string = self._exc_info_to_string(err, test) @@ -274,7 +269,7 @@ def addExpectedFailure(self, test, err): test (object): test item instance. err (tuple): tuple of values as returned by sys.exc_info(). """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addExpectedFailure(test, err) exception_string = self._exc_info_to_string(err, test) @@ -294,7 +289,7 @@ def addUnexpectedSuccess(self, test): test (object): test item instance. err (tuple): tuple of values as returned by sys.exc_info(). """ - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: super(Result, self).addUnexpectedSuccess(test) test.logger.error("Test %r ended in an unexpected success", test.data) diff --git a/src/rotest/core/runners/multiprocess/manager/message_handler.py b/src/rotest/core/runners/multiprocess/manager/message_handler.py index f2ff7329..7baf6016 100644 --- a/src/rotest/core/runners/multiprocess/manager/message_handler.py +++ b/src/rotest/core/runners/multiprocess/manager/message_handler.py @@ -9,7 +9,6 @@ from rotest.core.models.case_data import TestOutcome from rotest.core.models.general_data import GeneralData from rotest.management.common.parsers import DEFAULT_PARSER -from rotest.core.flow_component import AbstractFlowComponent from rotest.core.runners.multiprocess.common import (WrappedException, get_item_by_id) from rotest.management.common.messages import (StopTest, @@ -179,8 +178,11 @@ def _handle_start_message(self, test, message): test (object): test item to update. message (StartTest): worker message object. """ + # Since the logger is only created in the worker, we create it here + test.create_logger() + self.result.startTest(test) - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: self.runner.update_worker(worker_pid=message.msg_id, test=test) self.runner.update_timeout(worker_pid=message.msg_id, timeout=test.TIMEOUT) @@ -238,7 +240,7 @@ def _handle_stop_message(self, test, message): message (StopTest): worker message object. """ self.result.stopTest(test) - if not isinstance(test, AbstractFlowComponent) or test.is_main: + if test.is_main: self._update_parent_stop(test) def _handle_composite_stop_message(self, test, message): diff --git a/src/rotest/core/suite.py b/src/rotest/core/suite.py index 8d83e49a..d0fc9c30 100644 --- a/src/rotest/core/suite.py +++ b/src/rotest/core/suite.py @@ -1,9 +1,10 @@ """Define Rotest's TestSuite, composed from test suites or test cases.""" # pylint: disable=method-hidden,bad-super-call,too-many-arguments +# pylint: disable=too-many-locals from __future__ import absolute_import import unittest -from itertools import count +from itertools import count, chain from future.builtins import next @@ -11,6 +12,7 @@ from rotest.core.case import TestCase from rotest.core.flow import TestFlow from rotest.common.utils import get_work_dir +from rotest.core.result.result import Result from rotest.common.config import ROTEST_WORK_DIR from rotest.core.models.suite_data import SuiteData @@ -41,14 +43,16 @@ class TestSuite(unittest.TestSuite): _cleanup = False - def __init__(self, base_work_dir=ROTEST_WORK_DIR, save_state=True, - config=None, indexer=count(), parent=None, run_data=None, - enable_debug=False, skip_init=False, resource_manager=None): + def __init__(self, tests=(), indexer=count(), + base_work_dir=ROTEST_WORK_DIR, save_state=True, config=None, + parent=None, run_data=None, enable_debug=False, + skip_init=False, resource_manager=None): """Initialize 'components' & add them to the suite. Validates & initializes the TestSuite components & data object. Args: + tests (iterable): tests to add to the suite. base_work_dir (str): the base directory of the tests. save_state (bool): flag to determine if storing the states of resources is required. @@ -82,14 +86,14 @@ def __init__(self, base_work_dir=ROTEST_WORK_DIR, save_state=True, parent.addTest(self) core_log.debug("Initializing %r test-suite", name) - if len(self.components) == 0: + if len(self.components) == 0 and len(tests) == 0: raise AttributeError("%s: Components tuple can't be empty" % name) core_log.debug("Creating database entry for %r test-suite", name) self.work_dir = get_work_dir(base_work_dir, name, self) self.data = SuiteData(name=name, run_data=run_data) - for test_component in self.components: + for test_component in chain(self.components, tests): if issubclass(test_component, TestCase): for method_name in test_component.load_test_method_names(): @@ -121,14 +125,14 @@ def __init__(self, base_work_dir=ROTEST_WORK_DIR, save_state=True, elif issubclass(test_component, TestSuite): test_item = test_component(parent=self, - config=config, - indexer=indexer, - run_data=run_data, - skip_init=skip_init, - save_state=save_state, - enable_debug=enable_debug, - base_work_dir=self.work_dir, - resource_manager=resource_manager) + config=config, + indexer=indexer, + run_data=run_data, + skip_init=skip_init, + save_state=save_state, + enable_debug=enable_debug, + base_work_dir=self.work_dir, + resource_manager=resource_manager) core_log.debug("Adding %r to %r", test_item, self.data) @@ -165,12 +169,14 @@ def run(self, result, debug=False): rotest.core.result.result.Result. holder for test result information. """ - result.startComposite(self) + if isinstance(result, Result): + result.startComposite(self) core_log.debug("Running %r test-suite", self.data) result = super(TestSuite, self).run(result, debug) - result.stopComposite(self) + if isinstance(result, Result): + result.stopComposite(self) return result