Workflow API support

This patch introduces initial necessary changes for the Workflow API
support. The Workflow API allows the actor to specify APIs to use and it
will not to have to specify anymore messages to consume or produce.

With this, it is possible to implement pure code API functions for actor
writers, where they do not have to work with messages but directly can
work with data.

This allows us to provide a stable API that is not changed, however we
can change the messages and system below.
Additionally we can use multiple versions of the API where we can allow
to change APIs in newer versions to make them more useful to consumers
of the API, however we could keep the old versions around to keep their
original code working.

Signed-off-by: Vinzenz Feenstra <vfeenstr@redhat.com>

leapp/actors/__init__.py

@@ -9,9 +9,11 @@
     StopActorExecution, WorkflowConfigNotAvailable
 from leapp.models import DialogModel, Model
 from leapp.tags import Tag
+from leapp.utils import get_api_models
 from leapp.utils.i18n import install_translation_for_actor
 from leapp.utils.meta import get_flattened_subclasses
 from leapp.models.error_severity import ErrorSeverity
+from leapp.workflows.api import WorkflowAPI
 
 
 class Actor(object):
@@ -63,6 +65,12 @@ class Actor(object):
     Dialogs that are added to this list allow for persisting answers the user has given in the answer file storage.
     """
 
+    apis = ()
+    """
+    Tuple of :py:class:`leapp.workflow.api.WorkflowAPI` derived classes that implement Workflow APIs that are used by
+    an actor. Any models the apis produce or consume will be considered by the framework as if the actor defined them.
+    """
+
     text_domain = None
     """
     Using text domain allows to override the default gettext text domain, for custom localization support.
@@ -102,6 +110,9 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs=
         if config_model:
             self._configuration = next(self.consume(config_model), None)
 
+        # Needed so produce allows to send messages for models specified also by workflow APIs
+        type(self).produces = get_api_models(type(self), 'produces')
+
     def get_answers(self, dialog):
         """
         Gets the answers for a dialog. The dialog needs be predefined in :py:attr:`dialogs`.
@@ -450,8 +461,22 @@ def _is_tag_tuple(actor, name, value):
     return value
 
 
-def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info=''):
-    value = getattr(actor, name, None)
+def _is_api_tuple(actor, name, value):
+    if isinstance(value, type) and issubclass(value, WorkflowAPI):
+        _lint_warn(actor, name, "Apis")
+        value = (value,)
+    _is_type(tuple)(actor, name, value)
+    if not all([True] + [isinstance(item, type) and issubclass(item, WorkflowAPI) for item in value]):
+        raise WrongAttributeTypeError(
+            'Actor {} attribute {} should contain only WorkflowAPIs'.format(actor, name))
+    return value
+
+
+def _get_attribute(actor, name, validator, required=False, default_value=None, additional_info='', resolve=None):
+    if resolve:
+        value = resolve(actor, name)
+    else:
+        value = getattr(actor, name, None)
     if not value and required:
         raise MissingActorAttributeError('Actor {} is missing attribute {}.{}'.format(actor, name, additional_info))
     if value or required:
@@ -478,11 +503,12 @@ def get_actor_metadata(actor):
         ('path', os.path.dirname(os.path.realpath(sys.modules[actor.__module__].__file__))),
         _get_attribute(actor, 'name', _is_type(string_types), required=True),
         _get_attribute(actor, 'tags', _is_tag_tuple, required=True, additional_info=additional_tag_info),
-        _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=()),
-        _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=()),
+        _get_attribute(actor, 'consumes', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
+        _get_attribute(actor, 'produces', _is_model_tuple, required=False, default_value=(), resolve=get_api_models),
         _get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()),
         _get_attribute(actor, 'description', _is_type(string_types), required=False,
-                       default_value=actor.__doc__ or 'There has been no description provided for this actor.')
+                       default_value=actor.__doc__ or 'There has been no description provided for this actor.'),
+        _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=())
     ])
 
 

leapp/messaging/__init__.py

@@ -13,6 +13,7 @@
 from leapp.exceptions import CannotConsumeErrorMessages
 from leapp.models import DialogModel, ErrorModel
 from leapp.messaging.commands import WorkflowCommand
+from leapp.utils import get_api_models
 
 
 class BaseMessaging(object):
@@ -241,7 +242,8 @@ def consume(self, actor, *types):
         """
         types = tuple((getattr(t, '_resolved', t) for t in types))
         messages = list(self._data) + list(self._new_data)
-        lookup = {model.__name__: model for model in type(actor).consumes + self._config_models}
+        # Needs to use get_api_models to consider all consumes including the one specified by Workflow APIs
+        lookup = {model.__name__: model for model in get_api_models(type(actor), 'consumes') + self._config_models}
         if types:
             filtered = set(requested.__name__ for requested in types)
             messages = [message for message in messages if message['type'] in filtered]

leapp/repository/__init__.py

@@ -143,6 +143,7 @@ def load(self, resolve=True, stage=None):
             self._extend_environ_paths('LEAPP_COMMON_FILES', self.files)
             self.log.debug("Installing repository provided common libraries loader hook")
             sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.libraries.common', paths=self.libraries))
+            sys.meta_path.append(LeappLibrariesFinder(module_prefix='leapp.workflows.api', paths=self.apis))
 
         if not stage or stage is _LoadStage.ACTORS:
             self.log.debug("Running actor discovery")
@@ -188,6 +189,7 @@ def mapped_actor_data(data):
         return {
             'repo_dir': self._repo_dir,
             'actors': [mapped_actor_data(a.serialize()) for a in self.actors],
+            'apis': [dict([('path', path)]) for path in self.relative_paths(self.apis)],
             'topics': filtered_serialization(get_topics, self.topics),
             'models': filtered_serialization(get_models, self.models),
             'tags': filtered_serialization(get_tags, self.tags),
@@ -214,6 +216,13 @@ def actors(self):
         """
         return tuple(self._definitions.get(DefinitionKind.ACTOR, ()))
 
+    @property
+    def apis(self):
+        """
+        :return: Tuple of apis in the repository
+        """
+        return tuple(self._definitions.get(DefinitionKind.API, ()))
+
     @property
     def topics(self):
         """

leapp/repository/actor_definition.py

@@ -135,6 +135,7 @@ def serialize(self):
             'tags': self.tags,
             'consumes': self.consumes,
             'produces': self.produces,
+            'apis': self.apis,
             'dialogs': [dialog.serialize() for dialog in self.dialogs],
             'tools': self.tools,
             'files': self.files,
@@ -283,6 +284,13 @@ def injected_context(self):
             else:
                 os.environ.pop('LEAPP_TOOLS', None)
 
+    @property
+    def apis(self):
+        """
+        :return: names of APIs used by this actor
+        """
+        return tuple(self.discover()['apis'])
+
     @property
     def directory(self):
         """

leapp/repository/definition.py

@@ -15,6 +15,7 @@ def __init__(self, kind):
     TOOLS = _Kind('tools')
     FILES = _Kind('files')
     TESTS = _Kind('tests')
+    API = _Kind('api')
 
-    REPO_WHITELIST = (ACTOR, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES)
+    REPO_WHITELIST = (ACTOR, API, MODEL, TOPIC, TAG, WORKFLOW, TOOLS, LIBRARIES, FILES)
     ACTOR_WHITELIST = (TOOLS, LIBRARIES, FILES, TESTS)

leapp/repository/scan.py

@@ -90,7 +90,8 @@ def scan(repository, path):
         ('files', scan_files),
         ('libraries', scan_libraries),
         ('tests', scan_tests),
-        ('tools', scan_tools))
+        ('tools', scan_tools),
+        ('apis', scan_apis))
 
     dirs = [e for e in os.listdir(path) if os.path.isdir(os.path.join(path, e))]
     for name, task in scan_tasks:
@@ -251,3 +252,18 @@ def scan_tests(repo, path, repo_path):
     """
     if os.listdir(path):
         repo.add(DefinitionKind.TESTS, os.path.relpath(path, repo_path))
+
+
+def scan_apis(repo, path, repo_path):
+    """
+    Scans apis and adds them to the repository.
+
+    :param repo: Instance of the repository
+    :type repo: :py:class:`leapp.repository.Repository`
+    :param path: path to the apis
+    :type path: str
+    :param repo_path: path to the repository
+    :type repo_path: str
+    """
+    if os.listdir(path):
+        repo.add(DefinitionKind.API, os.path.relpath(path, repo_path))

leapp/snactor/__init__.py

@@ -44,6 +44,11 @@ def _load_commands_from(path):
 def cli(args):
     if args.logger_config and os.path.isfile(args.logger_config):
         os.environ['LEAPP_LOGGER_CONFIG'] = args.logger_config
+    # Consider using the in repository $REPOPATH/.leapp/logger.conf to actually obey --debug / --verbose
+    # If /etc/leapp/logger.conf or $REPOPATH/.leapp/logger.conf don't exist logging won't work in snactor.
+    elif find_repository_basedir('.') and os.path.isfile(os.path.join(find_repository_basedir('.'),
+                                                                      '.leapp/logger.conf')):
+        os.environ['LEAPP_LOGGER_CONFIG'] = os.path.join(find_repository_basedir('.'), '.leapp/logger.conf')
 
     config_file_path = None
     if args.config and os.path.isfile(args.config):

leapp/snactor/commands/discover.py

@@ -40,8 +40,9 @@ def _get_actor_path(actor, repository_relative=True):
 
 def _get_actor_details(actor):
     meta = actor.discover()
-    meta['produces'] = tuple(model.__name__ for model in meta['produces'])
-    meta['consumes'] = tuple(model.__name__ for model in meta['consumes'])
+    meta['produces'] = tuple(model.__name__ for model in actor.produces)
+    meta['consumes'] = tuple(model.__name__ for model in actor.consumes)
+    meta['apis'] = tuple(api.serialize() for api in actor.apis)
     meta['tags'] = tuple(tag.name for tag in meta['tags'])
     meta['path'] = _get_class_file(actor)
     meta['dialogs'] = [dialog.serialize() for dialog in actor.dialogs]

leapp/utils/__init__.py

@@ -3,3 +3,21 @@
 
 def reboot_system():
     subprocess.Popen(['/sbin/shutdown', '-r', 'now'])
+
+
+def get_api_models(actor, what):
+    """
+    Used to retrieve the full list of models including the ones defined by WorkflowAPIs used by the actor.
+
+    :param what: A string which either is 'consumes' or 'produces'
+    :type what: str
+    :param actor: Actor type/instance or ActorDefinition instance to retrieve the information from
+    :type actor: Actor or ActorDefinition
+    :return: Tuple of all produced or consumed models as specified by actor and APIs used by the actor.
+    """
+    def _do_get(api):
+        result = getattr(api, what, ())
+        for a in api.apis or ():
+            result = result + _do_get(a)
+        return result
+    return tuple(set(_do_get(actor)))

leapp/workflows/api/__init__.py

@@ -0,0 +1,17 @@
+class WorkflowAPI(object):
+    produces = ()
+    consumes = ()
+    apis = ()
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def serialize(cls):
+        return {
+            'name': cls.__name__,
+            'module': cls.__module__,
+            'consumes': [model.__name__ for model in cls.consumes],
+            'produces': [model.__name__ for model in cls.produces],
+            'apis': [api.__name__ for api in cls.apis]
+        }

tests/data/workflow-api-tests/.leapp/info

@@ -0,0 +1 @@
+{"name": "workflow-api-tests", "id": "e8bfeffe-9131-4792-bcc3-760628e0fc4b"}

tests/data/workflow-api-tests/.leapp/leapp.conf

@@ -0,0 +1,6 @@
+
+[repositories]
+repo_path=${repository:root_dir}
+
+[database]
+path=${repository:state_dir}/leapp.db

tests/data/workflow-api-tests/.leapp/logger.conf

@@ -0,0 +1,33 @@
+[loggers]
+keys=urllib3,root
+
+[formatters]
+keys=leapp
+
+[handlers]
+keys=leapp_audit,stream
+
+[formatter_leapp]
+format=%(asctime)s.%(msecs)-3d %(levelname)-8s PID: %(process)d %(name)s: %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
+
+[logger_urllib3]
+level=WARN
+qualname=urllib3
+handlers=stream
+
+[logger_root]
+level=DEBUG
+handlers=leapp_audit,stream
+
+[handler_leapp_audit]
+class=leapp.logger.LeappAuditHandler
+formatter=leapp
+args=()
+
+[handler_stream]
+class=StreamHandler
+level=ERROR
+formatter=leapp
+args=(sys.stderr,)

tests/data/workflow-api-tests/actors/apiv1test/actor.py

@@ -0,0 +1,30 @@
+from leapp.actors import Actor
+from leapp.exceptions import StopActorExecutionError
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v1 import TestAPI
+
+
+class ApiV1Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v1_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here also is however a, b, c
+        if api.order_change(1, 2, 3) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV1Test") != "survivor.v3.APIV1Test":
+            raise StopActorExecutionError("v1 survivor test failure")
+
+        # Ensure removed is available and works (Doesn't really do anything just ensures not to have an exception)
+        api.removed("ApiV1Test calls removed method")

tests/data/workflow-api-tests/actors/apiv2test/actor.py

@@ -0,0 +1,26 @@
+from leapp.actors import Actor
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v2 import TestAPI
+
+
+class ApiV2Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v2_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here is however c, b, a
+        if api.order_change(3, 2, 1) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV2Test") != "survivor.v3.APIV2Test":
+            raise StopActorExecutionError("v2 survivor test failure")

tests/data/workflow-api-tests/actors/apiv3test/actor.py

@@ -0,0 +1,27 @@
+from leapp.actors import Actor
+from leapp.exceptions import StopActorExecutionError
+from leapp.tags import FirstPhaseTag, WorkflowApiTestWorkflowTag
+from leapp.workflows.api.v3 import TestAPI
+
+
+class ApiV3Test(Actor):
+    """
+    This actor will check that the testing API ApiV3Test is behaving as expected.
+    """
+
+    name = 'api_v3_test'
+    consumes = ()
+    produces = ()
+    apis = (TestAPI,)
+    tags = (FirstPhaseTag, WorkflowApiTestWorkflowTag)
+
+    def process(self):
+        api = TestAPI()
+        # Expected output is always the order a, b, c
+        # The parameter order here is however b, c, a
+        if api.order_change(2, 3, 1) != (1, 2, 3):
+            raise StopActorExecutionError("order change API failure")
+
+        # Expected is that the input is prefixed with 'survivor.v3.'
+        if api.survivor("APIV3Test") != "survivor.v3.APIV3Test":
+            raise StopActorExecutionError("v3 survivor test failure")