Merge pull request #82 from gregoil/resources_discover_shell_mock

Resources discoverer and shell

setup.py

@@ -3,7 +3,7 @@
 
 from setuptools import setup, find_packages
 
-__version__ = "3.1.1"
+__version__ = "3.2.0"
 
 result_handlers = [
     "db = rotest.core.result.handlers.db_handler:DBHandler",

src/rotest/cli/main.py

@@ -3,10 +3,13 @@
 
 from rotest.cli.server import server
 from rotest.cli.client import main as run
+from rotest.management.utils.shell import main as shell
 
 
 def main():
     if len(sys.argv) > 1 and sys.argv[1] == "server":
         server()
+    elif len(sys.argv) > 1 and sys.argv[1] == "shell":
+        shell()
     else:
         run()

src/rotest/common/config.py

@@ -214,6 +214,12 @@ def get_configuration(configuration_schema,
         environment_variables=["ROTEST_SERVER_PORT"],
         config_file_options=["port"],
         default_value="7777"),
+    "discoverer_blacklist": Option(
+        config_file_options=["discoverer_blacklist"],
+        default_value=[]),
+    "shell_apps": Option(
+        config_file_options=["shell_apps"],
+        default_value=[]),
     "resource_request_timeout": Option(
         command_line_options=["--resource-request-timeout"],
         environment_variables=["ROTEST_RESOURCE_REQUEST_TIMEOUT",
@@ -252,6 +258,8 @@ def get_configuration(configuration_schema,
 RESOURCE_REQUEST_TIMEOUT = int(CONFIGURATION.resource_request_timeout)
 DJANGO_SETTINGS_MODULE = CONFIGURATION.django_settings
 ARTIFACTS_DIR = os.path.expanduser(CONFIGURATION.artifacts_dir)
+DISCOVERER_BLACKLIST = CONFIGURATION.discoverer_blacklist
+SHELL_APPS = CONFIGURATION.shell_apps
 
 if DJANGO_SETTINGS_MODULE is None:
     raise ValueError("No Django settings module was supplied")

src/rotest/core/block.py

@@ -152,7 +152,7 @@ def _share_outputs(self):
 
         self.share_data(**outputs_dict)
 
-    def _validate_inputs(self, extra_inputs=[]):
+    def validate_inputs(self, extra_inputs=[]):
         """Validate that all the required inputs of the blocks were passed.
 
         All names under the 'inputs' list must be attributes of the test-block

src/rotest/core/flow.py

@@ -121,15 +121,15 @@ def __init__(self, base_work_dir=ROTEST_WORK_DIR, save_state=True,
         self._set_parameters(override_previous=False, **self.__class__.common)
 
         if self.is_main:
-            self._validate_inputs()
+            self.validate_inputs()
 
     def __iter__(self):
         return iter(self._tests)
 
     def addTest(self, test_item):
         self._tests.append(test_item)
 
-    def _validate_inputs(self, extra_inputs=[]):
+    def validate_inputs(self, extra_inputs=[]):
         """Validate that all the required inputs of the blocks were passed.
 
         All names under the 'inputs' list must be attributes of the test-blocks
@@ -145,7 +145,7 @@ def _validate_inputs(self, extra_inputs=[]):
         fields = [request.name for request in self.get_resource_requests()]
         fields.extend(extra_inputs)
         for block in self:
-            block._validate_inputs(fields)
+            block.validate_inputs(fields)
             if isinstance(block, TestBlock):
                 fields.extend(block.get_outputs().keys())
 

src/rotest/core/flow_component.py

@@ -232,7 +232,7 @@ def setup_method_wrapper(*args, **kwargs):
 
                 if not self.is_main:
                     # Validate all required inputs were passed
-                    self._validate_inputs()
+                    self.validate_inputs()
 
                 setup_method(*args, **kwargs)
                 self.result.setupFinished(self)
@@ -381,7 +381,7 @@ def _set_parameters(self, override_previous=True, **parameters):
 
                     setattr(self, name, value)
 
-    def _validate_inputs(self, extra_inputs=[]):
+    def validate_inputs(self, extra_inputs=[]):
         """Validate that all the required inputs of the component were passed.
 
         Args:

src/rotest/management/utils/resources_discoverer.py

@@ -0,0 +1,73 @@
+"""Auxiliary module for discovering Rotest resources within an app."""
+import os
+from fnmatch import fnmatch
+
+import py
+import django
+from rotest.common.config import DISCOVERER_BLACKLIST
+from rotest.management.base_resource import BaseResource
+
+
+FILES_PATTERN = "*.py"
+
+
+def _is_resource_class(item):
+    """Check whether or not an object is a Rotest resource class.
+
+    Args:
+        item (object): object to check.
+
+    Returns:
+        bool. True if the given object is a Rotest resource, False otherwise.
+    """
+    return (isinstance(item, type) and issubclass(item, BaseResource) and
+            item is not BaseResource)
+
+
+def _import_resources_from_module(module_path):
+    """Return all resources in the module.
+
+    Args:
+        module_path (str): relative path to the module.
+
+    Returns:
+        dict. all resource classes found in the module {name: class}.
+    """
+    if not fnmatch(module_path, FILES_PATTERN):
+        return
+
+    module = py.path.local(module_path).pyimport()
+
+    return {item.__name__: item for item in module.__dict__.values()
+            if _is_resource_class(item)}
+
+
+def get_all_resources(app_name, blacklist=DISCOVERER_BLACKLIST):
+    """Get all the resource classes under a Django app.
+
+    Args:
+        app_name (str): application to search for resources inside.
+        blacklist (tuple): module patterns to ignore.
+
+    Returns:
+        dict. all resource classes found in the application {name: class}.
+    """
+    django.setup()
+    app_configs = django.apps.apps.app_configs
+    if app_name not in app_configs:
+        raise RuntimeError("Application %r was not found" % app_name)
+
+    app_path = app_configs[app_name].path
+    resources = {}
+
+    for sub_dir, _, modules in os.walk(app_path):
+        for module_name in modules:
+            module_path = os.path.join(sub_dir, module_name)
+            if any(fnmatch(module_path, pattern) for pattern in blacklist):
+                continue
+
+            module_resources = _import_resources_from_module(module_path)
+            if module_resources is not None:
+                resources.update(module_resources)
+
+    return resources

src/rotest/management/utils/shell.py

@@ -0,0 +1,115 @@
+"""Rotest shell module, which enables using resources and running blocks."""
+# pylint: disable=protected-access
+import sys
+
+import django
+import IPython
+from attrdict import AttrDict
+from rotest.common.config import SHELL_APPS
+from rotest.core.result.result import Result
+from rotest.management.base_resource import BaseResource
+from rotest.management.client.manager import ClientResourceManager
+from rotest.core.runner import parse_config_file, DEFAULT_CONFIG_PATH
+from rotest.management.utils.resources_discoverer import get_all_resources
+from rotest.core.result.handlers.stream.log_handler import LogDebugHandler
+
+
+# Mock tests result object for running blocks
+blocks_result = Result(stream=None, descriptions=None,
+                       outputs=[], main_test=None)
+
+
+# Mock tests configuration for running blocks
+blocks_config = AttrDict(parse_config_file(DEFAULT_CONFIG_PATH))
+
+
+# Container for data shared between blocks
+shared_data = {}
+
+
+ENABLE_DEBUG = False
+
+
+class ShellMockFlow(object):
+    """Mock class used for sharing data between blocks."""
+    parents_count = 0
+
+    def __init__(self):
+        self._tests = []
+        self.parent = None
+        self.identifier = 0
+        self.work_dir = None
+
+    def _set_parameters(self, override_previous=True, **kwargs):
+        shared_data.update(kwargs)
+        for test_item in self._tests:
+            test_item._set_parameters(override_previous, **kwargs)
+
+    def __iter__(self):
+        return iter([])
+
+    def addTest(self, test):
+        self._tests.append(test)
+
+    def request_resources(self, resources_to_request, *args, **kwargs):
+        for test_item in self._tests:
+            test_item.request_resources(resources_to_request, *args, **kwargs)
+            request_names = [request.name for request in resources_to_request]
+            test_item.share_data(**{name: getattr(test_item, name)
+                                    for name in request_names})
+
+
+def run_block(block_class, **kwargs):
+    """Run a block of the given class, passing extra parameters as arguments.
+
+    Args:
+        block_class (type): class inheriting from AbstractFlowComponent.
+        kwargs (dict): additional arguments that will be passed as parameters
+            to the block (overriding shared data).
+    """
+    shared_kwargs = shared_data.copy()
+    shared_kwargs.update(kwargs)
+    parent = ShellMockFlow()
+    block_class = block_class.params(**shared_kwargs)
+
+    block = block_class(config=blocks_config,
+                        parent=parent,
+                        enable_debug=ENABLE_DEBUG,
+                        resource_manager=BaseResource._SHELL_CLIENT,
+                        is_main=False)
+
+    parent.work_dir = block.work_dir
+    block.validate_inputs()
+    block.run(blocks_result)
+
+
+def main():
+    django.setup()
+    print "Creating client"
+    BaseResource._SHELL_CLIENT = ClientResourceManager()
+    BaseResource._SHELL_CLIENT.connect()
+    LogDebugHandler(None, sys.stdout, None)  # Activate log to screen
+    header = """Done! You can now lock resources and run tests, e.g.
+    resource1 = ResourceClass.lock(skip_init=True, name='resource_name')
+    resource2 = ResourceClass.lock(name='resource_name', config='config.json')
+    shared_data['resource'] = resource1
+    run_block(ResourceBlock, parameter=5)
+    run_block(ResourceBlock.params(parameter=6), resource=resource2)
+    """
+    try:
+        for app_name in SHELL_APPS:
+            print "Loading resources from app", app_name
+            resources = get_all_resources(app_name)
+            globals().update(resources)
+
+        IPython.embed(header=header,
+                      module=sys.modules["__main__"],
+                      user_ns=globals())
+
+    finally:
+        print "Releasing locked resources..."
+        BaseResource._SHELL_CLIENT.disconnect()
+
+
+if __name__ == "__main__":
+    main()