Merge pull request #166 from gregoil/fix_shell_behavior_bugs

Fix shell behavior bugs

docs/cli_options/client_options.rst

@@ -369,7 +369,7 @@ To optimize the running time of tests, you can use options :option:`-p` or
 :option:`--processes` to run several work processes that can run tests
 separately.
 
-Any test have a ``TIMEOUT`` attribute (defaults to 30 minutes), and it will be
+Any test have a ``TIMEOUT`` attribute (defaults to 60 minutes), and it will be
 enforced only when spawning at least one worker process:
 
 .. code-block:: python
@@ -382,6 +382,16 @@ enforced only when spawning at least one worker process:
         def test(self):
             pass
 
+.. warning::
+
+    When running with multiprocess you can't use IPDBugger (--debug).
+
+.. warning::
+
+    It is **not** recommended using this option when you have a SQLite database,
+    since it doesn't allow parallel access (all the workers request resources in the same time).
+
+
 Specifying Resources to Use
 ============================
 

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 
-__version__ = "7.13.0"
+__version__ = "7.13.1"
 
 result_handlers = [
     "db = rotest.core.result.handlers.db_handler:DBHandler",

src/rotest/cli/client.py

@@ -55,14 +55,16 @@
 
 django.setup()  # noqa
 
-from rotest.core.suite import TestSuite
 from rotest.common import core_log
+from rotest.core.suite import TestSuite
 from rotest.core.filter import match_tags
+from rotest.common.utils import parse_config_file
 from rotest.core.utils.common import print_test_hierarchy
 from rotest.core.result.result import get_result_handlers
 from rotest.cli.discover import discover_tests_under_paths
-from rotest.core.runner import (DEFAULT_CONFIG_PATH, parse_config_file,
-                                update_resource_requests, run as rotest_runner,
+from rotest.common.constants import (DEFAULT_CONFIG_PATH, DEFAULT_SCHEMA_PATH,
+                                     default_config)
+from rotest.core.runner import (update_resource_requests, run as rotest_runner,
                                 parse_resource_identifiers)
 
 
@@ -206,9 +208,11 @@ def main(*tests):
     parser = create_client_options_parser()
     arguments = parser.parse_args()
 
+    core_log.debug('Using configuration file %r', arguments.config_path)
     config = AttrDict(chain(
-        six.iteritems(parse_config_file(DEFAULT_CONFIG_PATH)),
-        six.iteritems(parse_config_file(arguments.config_path)),
+        six.iteritems(default_config),
+        six.iteritems(parse_config_file(arguments.config_path,
+                                        DEFAULT_SCHEMA_PATH)),
         filter_valid_values(vars(arguments)),
     ))
 

src/rotest/common/constants.py

@@ -1,4 +1,10 @@
 """This file contains constants for rotest."""
+import os
+
+from attrdict import AttrDict
+
+from .utils import parse_config_file
+
 BOLD = 'bold'
 DARK = 'dark'
 UNDERLINE = 'underline'
@@ -11,3 +17,9 @@
 GREEN = 'green'
 YELLOW = 'yellow'
 MAGENTA = 'magenta'
+
+FILE_FOLDER = os.path.dirname(__file__)
+DEFAULT_SCHEMA_PATH = os.path.join(FILE_FOLDER, "schema.json")
+DEFAULT_CONFIG_PATH = os.path.join(FILE_FOLDER, "default_config.json")
+default_config = AttrDict(parse_config_file(DEFAULT_CONFIG_PATH,
+                                            DEFAULT_SCHEMA_PATH))

src/rotest/common/utils.py

@@ -3,10 +3,13 @@
 from __future__ import absolute_import
 
 import os
+import json
 from shutil import copy
 from itertools import count
 from datetime import datetime
 
+from attrdict import AttrDict
+from jsonschema import validate
 from future.builtins import next
 
 RUNTIME_ORDER = '-start_time'
@@ -103,3 +106,48 @@ def get_class_fields(cls, field_type):
             field = getattr(cls, field_name)
             if isinstance(field, field_type):
                 yield (field_name, field)
+
+
+def parse_json(json_path, schema_path=None):
+    """Parse the Json file into attribute dictionary.
+
+    Args:
+        json_path (str): path of the Json file.
+        schema_path (str): path of the schema file - optional.
+
+    Returns:
+        AttrDict. representing the Json file .
+
+    Raises:
+        jsonschema.ValidationError: Json file does not comply with the schema.
+    """
+    with open(json_path) as config_file:
+        json_content = json.load(config_file)
+
+    if schema_path is not None:
+        with open(schema_path) as schema:
+            schema_content = json.load(schema)
+
+        validate(json_content, schema_content)
+
+    return AttrDict(json_content)
+
+
+def parse_config_file(json_path, schema_path):
+    """Parse configuration file to create the config dictionary.
+
+    Args:
+        json_path (str): path to the json config file.
+        schema_path (str): path of the schema file - optional.
+
+    Returns:
+        AttrDict. configuration dict, containing default values for run
+            options and other parameters.
+    """
+    if not os.path.exists(json_path):
+        raise ValueError("Illegal config-path: %r" % json_path)
+
+    config = parse_json(json_path=json_path,
+                        schema_path=schema_path)
+
+    return config

src/rotest/core/abstract_test.py

@@ -55,7 +55,7 @@ class AbstractTest(unittest.TestCase):
     SETUP_METHOD_NAME = 'setUp'
     TEARDOWN_METHOD_NAME = 'tearDown'
 
-    TIMEOUT = 1800  # 30 minutes
+    TIMEOUT = 60 * 60  # 60 minutes
 
     resources = ()
 

src/rotest/core/runner.py

@@ -2,14 +2,11 @@
 # pylint: disable=too-many-arguments
 from __future__ import absolute_import
 
-import os
 import sys
 from collections import defaultdict
 
 from future.builtins import range
 
-from rotest.common import core_log
-from rotest.core.utils.common import parse_json
 from rotest.common.utils import get_class_fields
 from rotest.core.runners.base_runner import BaseTestRunner
 from rotest.management.base_resource import ResourceRequest
@@ -22,9 +19,6 @@
 
 LAST_RUN_INDEX = -1
 MINIMUM_TIMES_TO_RUN = 1
-FILE_FOLDER = os.path.dirname(__file__)
-DEFAULT_SCHEMA_PATH = os.path.join(FILE_FOLDER, "schema.json")
-DEFAULT_CONFIG_PATH = os.path.join(FILE_FOLDER, "default_config.json")
 
 
 def get_runner(save_state=False, outputs=None, config=None,
@@ -129,27 +123,6 @@ def run(test_class, save_state=None, outputs=None, config=None,
     return runs_data
 
 
-def parse_config_file(json_path, schema_path=DEFAULT_SCHEMA_PATH):
-    """Parse configuration file to create the config dictionary.
-
-    Args:
-        json_path (str): path to the json config file.
-        schema_path (str): path of the schema file - optional.
-
-    Returns:
-        AttrDict. configuration dict, containing default values for run
-            options and other parameters.
-    """
-    if not os.path.exists(json_path):
-        raise ValueError("Illegal config-path: %r" % json_path)
-
-    core_log.debug('Parsing configuration file %r', json_path)
-    config = parse_json(json_path=json_path,
-                        schema_path=schema_path)
-
-    return config
-
-
 # Syntax symbol used to access the fields of Django models in querying
 SUBFIELD_ACCESSOR = '__'
 

src/rotest/core/utils/common.py

@@ -2,12 +2,8 @@
 # pylint: disable=dangerous-default-value
 from __future__ import print_function, absolute_import
 
-import json
 from future.builtins import str
 
-from attrdict import AttrDict
-from jsonschema import validate
-
 from rotest.core.case import TestCase
 from rotest.core.flow import TestFlow
 from rotest.core.block import TestBlock
@@ -85,28 +81,3 @@ def print_test_hierarchy(test, tag_filter, tags=[], depth=0):
             print_test_hierarchy(sub_test,
                                  PASS_ALL_FILTER if is_colored else None,
                                  tags, depth + 1)
-
-
-def parse_json(json_path, schema_path=None):
-    """Parse the Json file into attribute dictionary.
-
-    Args:
-        json_path (str): path of the Json file.
-        schema_path (str): path of the schema file - optional.
-
-    Returns:
-        AttrDict. representing the Json file .
-
-    Raises:
-        jsonschema.ValidationError: Json file does not comply with the schema.
-    """
-    with open(json_path) as config_file:
-        json_content = json.load(config_file)
-
-    if schema_path is not None:
-        with open(schema_path) as schema:
-            schema_content = json.load(schema)
-
-        validate(json_content, schema_content)
-
-    return AttrDict(json_content)

src/rotest/management/base_resource.py

@@ -20,8 +20,10 @@
 
 from rotest.common import core_log
 from rotest.common.config import ROTEST_WORK_DIR
+from rotest.common.utils import parse_config_file
 from rotest.common.utils import get_work_dir, get_class_fields
 from rotest.management.models.resource_data import ResourceData
+from rotest.common.constants import default_config, DEFAULT_SCHEMA_PATH
 
 try:
     from django.db.models.fields.related_descriptors import \
@@ -426,7 +428,6 @@ def lock(cls, config=None, skip_init=False, **kwargs):
             BaseResource. locked and initialized resource, ready for work.
         """
         # These runtime imports are done to avoid cyclic imports.
-        from rotest.core.runner import parse_config_file
         from rotest.management.client.manager import ClientResourceManager
 
         if BaseResource._SHELL_CLIENT is None:
@@ -436,13 +437,13 @@ def lock(cls, config=None, skip_init=False, **kwargs):
                                            cls,
                                            **kwargs)
 
-        config_dict = None
+        config_dict = default_config
         if config is not None:
             if isinstance(config, dict):
                 config_dict = config
 
             else:
-                config_dict = parse_config_file(config)
+                config_dict = parse_config_file(config, DEFAULT_SCHEMA_PATH)
 
         result = BaseResource._SHELL_CLIENT.request_resources(
                                                         [resource_request],

src/rotest/management/utils/shell.py

@@ -6,23 +6,19 @@
 
 import django
 import IPython
-from attrdict import AttrDict
 from future.builtins import object, next
 
 from rotest.core.suite import TestSuite
 from rotest.core.result.result import Result
+from rotest.common.constants import default_config
 from rotest.management.base_resource import BaseResource
 from rotest.core.flow_component import AbstractFlowComponent
 from rotest.management.client.manager import ClientResourceManager
-from rotest.core.runner import parse_config_file, DEFAULT_CONFIG_PATH
 from rotest.common.config import SHELL_STARTUP_COMMANDS, SHELL_OUTPUT_HANDLERS
 
 # Mock tests result object for running blocks
 result_object = None
 
-# Mock tests configuration for running blocks
-default_config = AttrDict(parse_config_file(DEFAULT_CONFIG_PATH))
-
 # Container for data shared between blocks
 shared_data = {}
 
@@ -73,18 +69,18 @@ def _run_block(block_class, config=default_config,
         kwargs (dict): additional arguments that will be passed as parameters
             to the block (overriding shared data).
     """
-    shared_kwargs = block_class.common.copy()
-    shared_kwargs.update(shared_data)
-    shared_kwargs.update(kwargs)
     parent = ShellMockFlow()
-    block_class = block_class.params(**shared_kwargs)
-
     block = block_class(config=config,
                         parent=parent,
                         enable_debug=debug,
                         resource_manager=BaseResource._SHELL_CLIENT,
                         is_main=False)
 
+    shared_kwargs = shared_data.copy()
+    shared_kwargs.update(kwargs)
+    block._set_parameters(override_previous=False,
+                          **shared_kwargs)
+
     parent.work_dir = block.work_dir
     block.validate_inputs()
     block.run(result_object)

tests/cli/test_client.py

@@ -12,7 +12,7 @@
 from rotest.core import TestCase, TestSuite
 from rotest.cli.client import main as client_main
 from rotest.cli.client import parse_outputs_option
-from rotest.core.runner import DEFAULT_SCHEMA_PATH, DEFAULT_CONFIG_PATH
+from rotest.common.constants import DEFAULT_SCHEMA_PATH, DEFAULT_CONFIG_PATH
 
 
 class MockCase(TestCase):