Add initial version of features mechanism

Introduce a potential mechanism for features in cibyl. Features
represent a new mechanism to query for information, particularly for
that information that can not be retrieved by a simple call combining
cli arguments.

Features are classes that inherit from the FeatureTemplate class and are
defined inside subpackages called 'features'. They must provide a
'query' method to be called, but the FeatureTemplate class provides on
that can be reused with slight tweaks in simple cases. The 'query' method
from  The FeatureTemplate can be overriden for any given Feature, it's
not mandatory to call it.

Currently there are two
locations for features a general (cibyl/features, empty) and an openstack
specific (cibyl/plugins/openstack/features, three features provided, HA,
IPV4 and IPV6). Plugins that add features must add their paths to the
FeatureLoader class when the plugin is enabled.

In the orchestrator the features are loaded and executed (calling the
query method). If more than one feature is requested, their results are
combined. If the user uses other cli arguments like --jobs, the
intersection of the jobs that satisfy each feature will be finally
published. If not, for each system it will be simply published whether
each feature is present or not.

The result of a Feature query without --jobs is to add a Feature model
to the system queried. This should not be confused with any particular
Feature like (HA or IPV4). The Feature model is a ci model (similar to
Job or System) stored in cibyl/models/ci/base/feature.py and serves as
store for the result of a Feature query.

cibyl/cli/main.py

@@ -115,7 +115,9 @@ def main():
         orchestrator.parser.parse()
         orchestrator.validate_environments()
         orchestrator.setup_sources()
-        orchestrator.query_and_publish(arguments["output_style"])
+        features = orchestrator.load_features()
+        orchestrator.query_and_publish(arguments["output_style"],
+                                       features=features)
     except CibylException as ex:
         if arguments["debug"]:
             raise ex

cibyl/cli/query.py

@@ -33,6 +33,10 @@ class QueryType(IntEnum):
     """Retrieve data concerning jobs and above."""
     BUILDS = 5
     """Retrieve data concerning builds and above."""
+    FEATURES = 6
+    """Retrieve data using features."""
+    FEATURES_JOBS = 7
+    """Retrieve data using features and jobs."""
 
 
 class QuerySelector:
@@ -75,6 +79,12 @@ def get_query_type_core(self, **kwargs):
         if build_args:
             result = QueryType.BUILDS
 
+        if 'features' in kwargs:
+            if job_args:
+                result = QueryType.FEATURES_JOBS
+            else:
+                result = QueryType.FEATURES
+
         return result
 
     def get_type_query(self, **kwargs):

cibyl/exceptions/__init__.py

@@ -26,4 +26,5 @@ def __init__(self, message=''):
         :param message: The reason for this error.
         :type message: str
         """
+        self.message = message
         super().__init__(*[message])

cibyl/features/__init__.py

@@ -0,0 +1,156 @@
+"""
+#    Copyright 2022 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+
+import logging
+import os
+import re
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from importlib.util import module_from_spec, spec_from_file_location
+from inspect import getmembers, isabstract, isclass
+
+from cibyl.exceptions.cli import InvalidArgument
+from cibyl.exceptions.source import NoSupportedSourcesFound, SourceException
+from cibyl.sources.source import (select_source_method,
+                                  source_information_from_method)
+from cibyl.utils.colors import Colors
+
+LOG = logging.getLogger(__name__)
+
+MODULE_PATTERN = re.compile(r"([a-zA-Z][a-zA-Z_]*)\.py")
+
+
+def is_feature_class(symbol):
+    """Check whether the symbols imported from a module correspond to
+    classes defining a feature. We assume that the symbol in question
+    defines a feature if it is a concrete class and is defined inside a module
+    within a subpackage name features.
+
+    :param symbol: An imported symbol to check
+    :type symbol: object
+    :returns: Whether the symbol defines a feature
+    :rtype: bool
+    """
+    is_concrete_class = isclass(symbol) and not isabstract(symbol)
+    return is_concrete_class and "features" in symbol.__module__
+
+
+class FeaturesLoader:
+    """Load all features available, either general or product specific through
+    some plugin."""
+    # __path__ is a list
+    features_locations = __path__
+
+    def __init__(self):
+        self.features_by_category = defaultdict(list)
+        self.all_features = {}
+        for location in self.features_locations:
+            for module_path in os.listdir(location):
+                module_match = MODULE_PATTERN.match(module_path)
+                if not module_match:
+                    continue
+                module_name = module_match.group(1)
+                # import the module directly from file, and add the _features
+                # suffix to the imported modules to help distinguish the
+                # classes defined there
+                # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
+                spec = spec_from_file_location(module_name+"_features",
+                                               f"{location}/{module_path}")
+                module = module_from_spec(spec)
+                spec.loader.exec_module(module)
+                features = getmembers(module, predicate=is_feature_class)
+                for feature_name, feature in features:
+                    self.all_features[feature_name.lower()] = feature
+                    self.features_by_category[module_name].append(feature_name)
+
+    def get_string_all_features(self):
+        """Get a string representation listing all available features to use in
+        an exception message."""
+        msg = ""
+        for category, features_names in self.features_by_category.items():
+            msg += f"\n{Colors.blue(category.title())}:"
+            for feature_name in features_names:
+                feature = self.all_features[feature_name.lower()]
+                docstring = getattr(feature, "__doc__", "")
+                msg += f"\n{Colors.red(' * ')}{Colors.blue(feature_name)}"
+                if docstring:
+                    msg += f"{Colors.red(' - '+feature.__doc__)}"
+        return msg
+
+    def get_feature(self, name_feature):
+        """Get the function associated with the given feature name
+
+        :param feature_name: Name of the feature
+        :type feature_name: str
+
+        :returns: class that implements the given feature
+        :rtype: class
+        :raises: InvalidArgument
+        """
+        try:
+            feature_class = self.all_features[name_feature.lower()]
+        except KeyError as err:
+            msg = f"No feature {name_feature}. Choose one of the following "
+            msg += "features:\n"
+            msg += self.get_string_all_features()
+            raise InvalidArgument(msg) from err
+        return feature_class()
+
+
+class FeatureTemplate(ABC):
+    """Skeleton for a generic feature."""
+    method_to_query = ""
+
+    @abstractmethod
+    def __init__(self, name):
+        self.name = name
+        self.args = {}
+
+    def query(self, system, **kwargs):
+        debug = kwargs.get("debug", False)
+        self.args.update(system.export_attributes_to_source())
+        self.args.update(kwargs)
+        try:
+            self.source_methods = select_source_method(system,
+                                                       self.method_to_query,
+                                                       **kwargs)
+        except NoSupportedSourcesFound as exception:
+            # if no sources are found in the system for this
+            # particular query, jump to the next one without
+            # stopping execution
+            LOG.error(exception, exc_info=debug)
+            return
+        query_result = {}
+        query_successful = False
+        for source_method, _ in self.source_methods:
+            try:
+                query_result = source_method(**self.args)
+                query_successful = True
+                break
+            except SourceException as exception:
+                source_info = source_information_from_method(
+                        source_method)
+                LOG.error("Error in %s with system %s. %s",
+                          source_info, system.name.value,
+                          exception, exc_info=debug)
+        if not query_successful:
+            msg = f"Feature {self.name} could not query any source for system "
+            msg += f"{system.name.value}"
+            LOG.warning(msg)
+            # use a return value of None to mark that no query was performed
+            return
+        system.register_query()
+        return query_result

cibyl/features/general.py

@@ -0,0 +1,15 @@
+"""
+#    Copyright 2022 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""

cibyl/models/ci/base/system.py

@@ -22,6 +22,7 @@
 from cibyl.models.attribute import AttributeDictValue, AttributeListValue
 from cibyl.models.ci.base.job import Job
 from cibyl.models.model import Model
+from cibyl.models.product.feature import Feature
 from cibyl.sources.source import Source
 
 
@@ -61,7 +62,16 @@ class System(Model):
         'queried': {
             'attr_type': bool,
             'arguments': []
-        }
+        },
+        'features': {
+            'attr_type': Feature,
+            'attribute_value_class': AttributeDictValue,
+            'arguments': [
+                Argument(name='--features', arg_type=str,
+                         nargs="*",
+                         description="Template to query for a given feature")
+            ]
+        },
     }
     """Defines the CLI arguments for all systems.
     """
@@ -155,6 +165,10 @@ def populate(self, instances):
         else:
             raise NonSupportedModelType(instances.attr_type)
 
+    def add_feature(self, feature):
+        """Add a feature to the system."""
+        self.features[feature.name.value] = feature
+
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return False

cibyl/models/product/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.

cibyl/models/product/feature.py

@@ -0,0 +1,39 @@
+"""
+# Copyright 2022 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+# pylint: disable=no-member
+from cibyl.models.model import Model
+
+
+class Feature(Model):
+    """Represents a Feature present (or not) in a CI environment."""
+
+    API = {
+        'name': {
+            'attr_type': str,
+            'arguments': []
+        },
+        'present': {
+            'attr_type': bool,
+            'arguments': []
+        },
+    }
+
+    def __init__(self, name, present):
+        # Let IDEs know this model's attributes
+        self.name = None
+        self.systems = None
+
+        super().__init__({'name': name, 'present': present})