Workflow API support

leapp/actors/__init__.py

@@ -9,9 +9,11 @@
     StopActorExecution, WorkflowConfigNotAvailable
 from leapp.models import DialogModel, Model
 from leapp.tags import Tag
+from leapp.utils import get_api_models
 from leapp.utils.i18n import install_translation_for_actor
 from leapp.utils.meta import get_flattened_subclasses
 from leapp.models.error_severity import ErrorSeverity
+from leapp.workflows.api import WorkflowAPI
 
 
 class Actor(object):
@@ -63,6 +65,12 @@ class Actor(object):
     Dialogs that are added to this list allow for persisting answers the user has given in the answer file storage.
     """
 
+    apis = ()
+    """
+    Tuple of :py:class:`leapp.workflow.api.WorkflowAPI` derived classes that implement Workflow APIs that are used by
+    an actor. Any models the apis produce or consume will be considered by the framework as if the actor defined them.
+    """
+
     text_domain = None
     """
     Using text domain allows to override the default gettext text domain, for custom localization support.
@@ -102,6 +110,9 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs=
         if config_model:
             self._configuration = next(self.consume(config_model), None)
 
+        # Needed so produce allows to send messages for models specified also by workflow APIs
+        type(self).produces = get_api_models(type(self), 'produces')
+
     def get_answers(self, dialog):
         """
         Gets the answers for a dialog. The dialog needs be predefined in :py:attr:`dialogs`.
@@ -450,8 +461,22 @@ def _is_tag_tuple(actor, name, value):
     return value
 
 
-def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info=''):
-    value = getattr(actor, name, None)
+def _is_api_tuple(actor, name, value):
+    if isinstance(value, type) and issubclass(value, WorkflowAPI):
+        _lint_warn(actor, name, "Apis")
+        value = (value,)
+    _is_type(tuple)(actor, name, value)
+    if not all([True] + [isinstance(item, type) and issubclass(item, WorkflowAPI) for item in value]):
+        raise WrongAttributeTypeError(
+            'Actor {} attribute {} should contain only WorkflowAPIs'.format(actor, name))
+    return value
+
+
+def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info='', resolve=None):
+    if resolve:
+        value = resolve(actor, name)
+    else:
+        value = getattr(actor, name, None)
     if not value and required:
         raise MissingActorAttributeError('Actor {} is missing attribute {}.{}'.format(actor, name, additional_info))
     if value or required:
@@ -478,11 +503,12 @@ def get_actor_metadata(actor):
         ('path', os.path.dirname(os.path.realpath(sys.modules[actor.__module__].__file__))),
         _get_attribute(actor, 'name', _is_type(string_types), required=True),
         _get_attribute(actor, 'tags', _is_tag_tuple, required=True, additional_info=additional_tag_info),
-        _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=()),
-        _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=()),
+        _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
+        _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
         _get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()),
         _get_attribute(actor, 'description', _is_type(string_types), required=False,
-                       default_value=actor.__doc__ or 'There has been no description provided for this actor.')
+                       default_value=actor.__doc__ or 'There has been no description provided for this actor.'),
+        _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=())
     ])
 
 

leapp/messaging/__init__.py

@@ -13,6 +13,7 @@
 from leapp.exceptions import CannotConsumeErrorMessages
 from leapp.models import DialogModel, ErrorModel
 from leapp.messaging.commands import WorkflowCommand
+from leapp.utils import get_api_models
 
 
 class BaseMessaging(object):
@@ -241,7 +242,8 @@ def consume(self, actor, *types):
         """
         types = tuple((getattr(t, '_resolved', t) for t in types))
         messages = list(self._data) + list(self._new_data)
-        lookup = {model.__name__: model for model in type(actor).consumes + self._config_models}
+        # Needs to use get_api_models to consider all consumes including the one specified by Workflow APIs
+        lookup = {model.__name__: model for model in get_api_models(type(actor), 'consumes') + self._config_models}
         if types:
             filtered = set(requested.__name__ for requested in types)
             messages = [message for message in messages if message['type'] in filtered]

leapp/repository/__init__.py

@@ -143,6 +143,7 @@ def load(self, resolve=True, stage=None):
             self._extend_environ_paths('LEAPP_COMMON_FILES', self.files)
             self.log.debug("Installing repository provided common libraries loader hook")
             sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.libraries.common', paths=self.libraries))
+            sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.workflows.api', paths=self.apis))
 
         if not stage or stage is _LoadStage.ACTORS:
             self.log.debug("Running actor discovery")
@@ -188,6 +189,7 @@ def mapped_actor_data(data):
         return {
             'repo_dir': self._repo_dir,
             'actors': [mapped_actor_data(a.serialize()) for a in self.actors],
+            'apis': [dict([('path', path)]) for path in self.relative_paths(self.apis)],
             'topics': filtered_serialization(get_topics, self.topics),
             'models': filtered_serialization(get_models, self.models),
             'tags': filtered_serialization(get_tags, self.tags),
@@ -214,6 +216,13 @@ def actors(self):
         """
         return tuple(self._definitions.get(DefinitionKind.ACTOR, ()))
 
+    @property
+    def apis(self):
+        """
+        :return: Tuple of apis in the repository
+        """
+        return tuple(self._definitions.get(DefinitionKind.API, ()))
+
     @property
     def topics(self):
         """

leapp/repository/actor_definition.py

@@ -135,6 +135,7 @@ def serialize(self):
             'tags': self.tags,
             'consumes': self.consumes,
             'produces': self.produces,
+            'apis': self.apis,
             'dialogs': [dialog.serialize() for dialog in self.dialogs],
             'tools': self.tools,
             'files': self.files,
@@ -283,6 +284,13 @@ def injected_context(self):
             else:
                 os.environ.pop('LEAPP_TOOLS', None)
 
+    @property
+    def apis(self):
+        """
+        :return: names of APIs used by this actor
+        """
+        return tuple(self.discover()['apis'])
+
     @property
     def directory(self):
         """

leapp/repository/definition.py

@@ -15,6 +15,7 @@ def __init__(self, kind):
     TOOLS = _Kind('tools')
     FILES = _Kind('files')
     TESTS = _Kind('tests')
+    API = _Kind('api')
 
-    REPO_WHITELIST = (ACTOR, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES)
+    REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES)
     ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, TESTS)

leapp/repository/scan.py

@@ -90,7 +90,8 @@ def scan(repository, path):
         ('files', scan_files),
         ('libraries', scan_libraries),
         ('tests', scan_tests),
-        ('tools', scan_tools))
+        ('tools', scan_tools),
+        ('apis', scan_apis))
 
     dirs = [e for e in os.listdir(path) if os.path.isdir(os.path.join(path, e))]
     for name, task in scan_tasks:
@@ -251,3 +252,18 @@ def scan_tests(repo, path, repo_path):
     """
     if os.listdir(path):
         repo.add(DefinitionKind.TESTS, os.path.relpath(path, repo_path))
+
+
+def scan_apis(repo, path, repo_path):
+    """
+    Scans apis and adds them to the repository.
+
+    :param repo: Instance of the repository
+    :type repo: :py:class:`leapp.repository.Repository`
+    :param path: path to the apis
+    :type path: str
+    :param repo_path: path to the repository
+    :type repo_path: str
+    """
+    if os.listdir(path):
+        repo.add(DefinitionKind.API, os.path.relpath(path, repo_path))

leapp/snactor/__init__.py

@@ -44,6 +44,11 @@ def _load_commands_from(path):
 def cli(args):
     if args.logger_config and os.path.isfile(args.logger_config):
         os.environ['LEAPP_LOGGER_CONFIG'] = args.logger_config
+    # Consider using the in repository $REPOPATH/.leapp/logger.conf to actually obey --debug / --verbose
+    # If /etc/leapp/logger.conf or $REPOPATH/.leapp/logger.conf don't exist logging won't work in snactor.
+    elif find_repository_basedir('.') and os.path.isfile(os.path.join(find_repository_basedir('.'),
+                                                                      '.leapp/logger.conf')):
+        os.environ['LEAPP_LOGGER_CONFIG'] = os.path.join(find_repository_basedir('.'), '.leapp/logger.conf')
 
     config_file_path = None
     if args.config and os.path.isfile(args.config):

leapp/snactor/commands/discover.py

@@ -40,8 +40,9 @@ def _get_actor_path(actor, repository_relative=True):
 
 def _get_actor_details(actor):
     meta = actor.discover()
-    meta['produces'] = tuple(model.__name__ for model in meta['produces'])
-    meta['consumes'] = tuple(model.__name__ for model in meta['consumes'])
+    meta['produces'] = tuple(model.__name__ for model in actor.produces)
+    meta['consumes'] = tuple(model.__name__ for model in actor.consumes)
+    meta['apis'] = tuple(api.serialize() for api in actor.apis)
     meta['tags'] = tuple(tag.name for tag in meta['tags'])
     meta['path'] = _get_class_file(actor)
     meta['dialogs'] = [dialog.serialize() for dialog in actor.dialogs]

leapp/utils/__init__.py

@@ -3,3 +3,21 @@
 
 def reboot_system():
     subprocess.Popen(['/sbin/shutdown', '-r', 'now'])
+
+
+def get_api_models(actor, what):
+    """
+    Used to retrieve the full list of models including the ones defined by WorkflowAPIs used by the actor.
+
+    :param what: A string which either is 'consumes' or 'produces'
+    :type what: str
+    :param actor: Actor type/instance or ActorDefinition instance to retrieve the information from
+    :type actor: Actor or ActorDefinition
+    :return: Tuple of all produced or consumed models as specified by actor and APIs used by the actor.
+    """
+    def _do_get(api):
+        result = getattr(api, what, ())
+        for a in api.apis or ():
+            result = result + _do_get(a)
+        return result
+    return tuple(set(_do_get(actor)))

leapp/workflows/api/__init__.py

@@ -0,0 +1,17 @@
+class WorkflowAPI(object):
+    produces = ()
+    consumes = ()
+    apis = ()
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def serialize(cls):
+        return {
+            'name': cls.__name__,
+            'module': cls.__module__,
+            'consumes': [model.__name__ for model in cls.consumes],
+            'produces': [model.__name__ for model in cls.produces],
+            'apis': [api.__name__ for api in cls.apis]
+        }

tests/data/workflow-api-tests/.leapp/info

@@ -0,0 +1 @@
+{"name": "workflow-api-tests", "id": "e8bfeffe-9131-4792-bcc3-760628e0fc4b"}

tests/data/workflow-api-tests/.leapp/leapp.conf

@@ -0,0 +1,6 @@
+
+[repositories]
+repo_path=${repository:root_dir}
+
+[database]
+path=${repository:state_dir}/leapp.db

tests/data/workflow-api-tests/.leapp/logger.conf

@@ -0,0 +1,33 @@
+[loggers]
+keys=urllib3,root
+
+[formatters]
+keys=leapp
+
+[handlers]
+keys=leapp_audit,stream
+
+[formatter_leapp]
+format=%(asctime)s.%(msecs)-3d %(levelname)-8s PID: %(process)d %(name)s: %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
+
+[logger_urllib3]
+level=WARN
+qualname=urllib3
+handlers=stream
+
+[logger_root]
+level=DEBUG
+handlers=leapp_audit,stream
+
+[handler_leapp_audit]
+class=leapp.logger.LeappAuditHandler
+formatter=leapp
+args=()
+
+[handler_stream]
+class=StreamHandler
+level=ERROR
+formatter=leapp
+args=(sys.stderr,)

tests/data/workflow-api-tests/actors/apiv1test/actor.py

@@ -0,0 +1,30 @@
+from leapp.actors import Actor
+from leapp.exceptions import StopActorExecutionError
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v1 import TestAPI
+
+
+class ApiV1Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v1_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here also is however a, b, c
+        if api.order_change(1, 2, 3) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV1Test") != "survivor.v3.APIV1Test":
+            raise StopActorExecutionError("v1 survivor test failure")
+
+        # Ensure removed is available and works (Doesn't really do anything just ensures not to have an exception)
+        api.removed("ApiV1Test calls removed method")

tests/data/workflow-api-tests/actors/apiv2test/actor.py

@@ -0,0 +1,26 @@
+from leapp.actors import Actor
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v2 import TestAPI
+
+
+class ApiV2Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v2_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here is however c, b, a
+        if api.order_change(3, 2, 1) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV2Test") != "survivor.v3.APIV2Test":
+            raise StopActorExecutionError("v2 survivor test failure")

tests/data/workflow-api-tests/actors/apiv3test/actor.py

@@ -0,0 +1,27 @@
+from leapp.actors import Actor
+from leapp.exceptions import StopActorExecutionError
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v3 import TestAPI
+
+
+class ApiV3Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v3_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here is however b, c, a
+        if api.order_change(2, 3, 1) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV3Test") != "survivor.v3.APIV3Test":
+            raise StopActorExecutionError("v3 survivor test failure")