diff --git a/powerapi/actor/actor.py b/powerapi/actor/actor.py index 74f96172..edf9e795 100644 --- a/powerapi/actor/actor.py +++ b/powerapi/actor/actor.py @@ -36,9 +36,8 @@ import traceback import setproctitle -from powerapi.exception import PowerAPIExceptionWithMessage +from powerapi.exception import PowerAPIExceptionWithMessage, UnknownMessageTypeException from powerapi.message import PoisonPillMessage -from powerapi.message import UnknownMessageTypeException from powerapi.handler import HandlerException from .socket_interface import SocketInterface @@ -93,7 +92,7 @@ def __init__(self, name, level_logger=logging.WARNING, timeout=None): :param str name: unique name that will be used to indentify the actor processus :param int level_logger: Define the level of the logger - :param int timeout: if define, do something if no msg is recv every + :param int timeout: if defined, do something if no msg is recv every timeout (in ms) """ multiprocessing.Process.__init__(self, name=name) @@ -229,7 +228,7 @@ def _initial_behaviour(self): handler = self.state.get_corresponding_handler(msg) handler.handle_message(msg) except UnknownMessageTypeException: - self.logger.warning("UnknowMessageTypeException: " + str(msg)) + self.logger.warning("UnknownMessageTypeException: " + str(msg)) except HandlerException: self.logger.warning("HandlerException") diff --git a/powerapi/actor/state.py b/powerapi/actor/state.py index 7c2f16e4..eb1b322c 100644 --- a/powerapi/actor/state.py +++ b/powerapi/actor/state.py @@ -27,7 +27,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.message import UnknownMessageTypeException +from powerapi.exception import UnknownMessageTypeException from powerapi.actor.supervisor import Supervisor diff --git a/powerapi/backend_supervisor/backend_supervisor.py b/powerapi/backend_supervisor/backend_supervisor.py index aa31b558..b761dcc9 100644 --- a/powerapi/backend_supervisor/backend_supervisor.py +++ b/powerapi/backend_supervisor/backend_supervisor.py @@ -32,10 +32,10 @@ from powerapi.actor import Supervisor from powerapi.puller import PullerActor from powerapi.dispatcher import DispatcherActor +from powerapi.pusher import PusherActor class BackendSupervisor(Supervisor): - """ Provide additional functionality to deal with actors: join """ @@ -55,6 +55,9 @@ def __init__(self, stream_mode): #: (list): List of Pusher self.pushers = [] + #: (list): List of pre processors + self.pre_processors = [] + def join(self): """ wait until all actor are terminated @@ -65,8 +68,10 @@ def join(self): self.pullers.append(actor) elif isinstance(actor, DispatcherActor): self.dispatchers.append(actor) - else: + elif isinstance(actor, PusherActor): self.pushers.append(actor) + else: + self.pre_processors.append(actor) if self.stream_mode: self.join_stream_mode_on() @@ -97,12 +102,16 @@ def join_stream_mode_off(self): """ Supervisor behaviour when stream mode is off. - Supervisor wait the Puller death + - Supervisor wait the pre-processors death - Supervisor wait for the dispatcher death - Supervisor send a PoisonPill (by_data) to the Pusher - Supervisor wait for the Pusher death """ for puller in self.pullers: puller.join() + for pre_processor in self.pre_processors: + pre_processor.soft_kill() + pre_processor.join() for dispatcher in self.dispatchers: dispatcher.soft_kill() dispatcher.join() diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py new file mode 100644 index 00000000..aac9a4aa --- /dev/null +++ b/powerapi/cli/binding_manager.py @@ -0,0 +1,233 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# pylint: disable=R1702 +from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed +from powerapi.processor.processor_actor import ProcessorActor +from powerapi.puller import PullerActor +from powerapi.pusher import PusherActor + + +class BindingManager: + """ + Class for management the binding between actors during their creation process + """ + + def __init__(self, actors: dict = {}): + """ + :param dict actors: Dictionary of actors to create the bindings. The name of the actor is the key + """ + if not actors: + self.actors = {} + else: + self.actors = actors + + def process_bindings(self): + """ + Define bindings between self.actors according to the processors' targets. + """ + raise NotImplementedError() + + +class ProcessorBindingManager(BindingManager): + """ + Class for management of bindings between processor actors and others actors + """ + + def __init__(self, actors: dict, processors: dict): + """ + The ProcessorBindingManager defines bindings between actors and processors + :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + + BindingManager.__init__(self, actors=actors) + if not processors: + self.processors = {} + else: + self.processors = processors + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + """ + for target_actor_name in processor.state.target_actors_names: + if target_actor_name not in self.actors: + raise UnexistingActorException(actor=target_actor_name) + + def check_processors_targets_are_unique(self): + """ + Check that processors targets are unique, i.e., the same target is not related to + two different processors + """ + used_targets = [] + for _, processor in self.processors.items(): + for target_actor_name in processor.state.target_actors_names: + if target_actor_name in used_targets: + raise TargetActorAlreadyUsed(target_actor=target_actor_name) + else: + used_targets.append(target_actor_name) + + +class PreProcessorBindingManager(ProcessorBindingManager): + """ + Class for management the binding between pullers and pre-processor actors + """ + + def __init__(self, pullers: dict, processors: dict): + """ + The PreProcessorBindingManager defines bindings between pullers and processors: puller->processor->dispatcher + :param dict pullers: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + + ProcessorBindingManager.__init__(self, actors=pullers, processors=processors) + + def process_bindings(self): + """ + Define bindings between self.actors according to the pre-processors' targets. + + """ + + # Check that processors targets are unique + self.check_processors_targets_are_unique() + + # For each processor, we get targets and create the binding: + # puller->processor->dispatcher + for _, processor in self.processors.items(): + + self.check_processor_targets(processor=processor) + + for target_actor_name in processor.state.target_actors_names: + + # The processor has to be between the puller and the dispatcher + # The dispatcher becomes a target of the processor + + puller_actor = self.actors[target_actor_name] + + # The dispatcher defines the relationship between the Formula and + # Puller + number_of_filters = len(puller_actor.state.report_filter.filters) + + for index in range(number_of_filters): + # The filters define the relationship with the dispatcher + # The relationship has to be updated + current_filter = list(puller_actor.state.report_filter.filters[index]) + current_filter_dispatcher = current_filter[1] + processor.add_target_actor(actor=current_filter_dispatcher) + current_filter[1] = processor + puller_actor.state.report_filter.filters[index] = tuple(current_filter) + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PullerActor instance. + If it is not the case, it raises UnsupportedActorTypeException + """ + ProcessorBindingManager.check_processor_targets(self, processor=processor) + + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] + + if not isinstance(actor, PullerActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) + + +class PostProcessorBindingManager(ProcessorBindingManager): + """ + Class for management the binding between post-processor and pusher actors + """ + + def __init__(self, pushers: dict, processors: dict, pullers: dict): + """ + The PostProcessorBindingManager defines bindings between processors and pushers: formula->processor->pushers + :param dict pushers: Dictionary of PusherActors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + ProcessorBindingManager.__init__(self, actors=pushers, processors=processors) + self.pullers = pullers + + def process_bindings(self): + """ + Define bindings between self.actors according to the post-processors' targets. + + """ + + # For each processor, we get targets and create the binding: + # formula->processor->pusher + for _, processor in self.processors.items(): + + self.check_processor_targets(processor=processor) + + for target_actor_name in processor.state.target_actors_names: + + # The processor has to be between the formula and the pusher + # The pusher becomes a target of the processor + + pusher_actor = self.actors[target_actor_name] + + processor.add_target_actor(actor=pusher_actor) + + # We look for the pusher on each dispatcher in order to replace it by + # the processor + for _, puller in self.pullers: + + for current_filter in puller.state.report_filter.filters: + dispatcher = current_filter[1] + + number_of_pushers = len(dispatcher.pusher) + pusher_updated = False + + for index in range(number_of_pushers): + if dispatcher.pusher[index] == pusher_actor: + dispatcher.pusher[index] = processor + pusher_updated = True + break + + if pusher_updated: + dispatcher.update_state_formula_factory() + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PusherActor instance. + If it is not the case, it raises UnsupportedActorTypeException + """ + ProcessorBindingManager.check_processor_targets(self, processor=processor) + + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] + + if not isinstance(actor, PusherActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index d7e9257c..8b95eb77 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -37,7 +37,8 @@ POWERAPI_ENVIRONMENT_VARIABLE_PREFIX = 'POWERAPI_' POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'OUTPUT_' POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'INPUT_' -POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'REPORT_MODIFIER_' +POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PRE_PROCESSOR_' +POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'POST_PROCESSOR' def extract_file_names(arg, val, args, acc): @@ -60,18 +61,21 @@ def __init__(self): self.add_argument_prefix(argument_prefix=POWERAPI_ENVIRONMENT_VARIABLE_PREFIX) # Subgroups - self.add_subgroup(name='report_modifier', - prefix=POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX, - help_text="Specify a report modifier to change input report values : " - "--report_modifier ARG1 ARG2 ...") - self.add_subgroup(name='input', prefix=POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a database input : --db_input database_name ARG1 ARG2 ... ") + help_text="specify a database input : --input database_name ARG1 ARG2 ... ") self.add_subgroup(name='output', prefix=POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a database output : --db_output database_name ARG1 ARG2 ...") + help_text="specify a database output : --output database_name ARG1 ARG2 ...") + + self.add_subgroup(name='pre-processor', + prefix=POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a pre-processor : --pre-processor pre_processor_name ARG1 ARG2 ...") + + self.add_subgroup(name='post-processor', + prefix=POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a post-processor : --post-processor post_processor_name ARG1 ARG2 ...") # Parsers @@ -92,21 +96,6 @@ def __init__(self): help_text="enable stream mode", ) - subparser_libvirt_mapper_modifier = SubgroupConfigParsingManager("libvirt_mapper") - subparser_libvirt_mapper_modifier.add_argument( - "u", "uri", help_text="libvirt daemon uri", default_value="" - ) - subparser_libvirt_mapper_modifier.add_argument( - "d", - "domain_regexp", - help_text="regexp used to extract domain from cgroup string", - ) - subparser_libvirt_mapper_modifier.add_argument("n", "name", help_text="") - self.add_subgroup_parser( - subgroup_name="report_modifier", - subgroup_parser=subparser_libvirt_mapper_modifier - ) - subparser_mongo_input = SubgroupConfigParsingManager("mongodb") subparser_mongo_input.add_argument("u", "uri", help_text="specify MongoDB uri") subparser_mongo_input.add_argument( @@ -423,6 +412,69 @@ def __init__(self): subgroup_parser=subparser_influx2_output ) + subparser_libvirt_pre_processor = SubgroupConfigParsingManager("libvirt") + subparser_libvirt_pre_processor.add_argument( + "u", "uri", help_text="libvirt daemon uri", default_value="" + ) + subparser_libvirt_pre_processor.add_argument( + "d", + "domain-regexp", + help_text="regexp used to extract domain from cgroup string", + ) + + subparser_libvirt_pre_processor.add_argument( + "p", + "puller", + help_text="target puller for the pre-processor", + ) + + subparser_libvirt_pre_processor.add_argument("n", "name", help_text="") + self.add_subgroup_parser( + subgroup_name="pre-processor", + subgroup_parser=subparser_libvirt_pre_processor + ) + + subparser_k8s_pre_processor = SubgroupConfigParsingManager("k8s") + subparser_k8s_pre_processor.add_argument( + "a", "k8s-api-mode", help_text="k8s api mode (local, manual or cluster)" + ) + subparser_k8s_pre_processor.add_argument( + "t", + "time-interval", + help_text="time interval for the k8s monitoring", + argument_type=int + ) + subparser_k8s_pre_processor.add_argument( + "o", + "timeout-query", + help_text="timeout for k8s queries", + argument_type=int + ) + + subparser_k8s_pre_processor.add_argument( + "k", + "api-key", + help_text="API key authorization required for k8s manual configuration", + ) + + subparser_k8s_pre_processor.add_argument( + "h", + "host", + help_text="host required for k8s manual configuration", + ) + + subparser_k8s_pre_processor.add_argument( + "p", + "puller", + help_text="target puller for the pre-processor", + ) + + subparser_k8s_pre_processor.add_argument("n", "name", help_text="") + self.add_subgroup_parser( + subgroup_name="pre-processor", + subgroup_parser=subparser_k8s_pre_processor + ) + def parse_argv(self): """ """ try: diff --git a/powerapi/cli/config_validator.py b/powerapi/cli/config_validator.py index 57fe087b..65b62a07 100644 --- a/powerapi/cli/config_validator.py +++ b/powerapi/cli/config_validator.py @@ -32,13 +32,15 @@ from typing import Dict -from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException +from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException, \ + UnexistingActorException class ConfigValidator: """ Validate powerapi config and initialize missing default values """ + @staticmethod def validate(config: Dict): """ @@ -66,7 +68,8 @@ def validate(config: Dict): for input_id in config['input']: input_config = config['input'][input_id] if input_config['type'] == 'csv' \ - and ('files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): + and ( + 'files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): logging.error("no files parameter found for csv input") raise MissingArgumentException(argument_name='files') @@ -79,10 +82,41 @@ def validate(config: Dict): if 'name' not in input_config: input_config['name'] = 'default_puller' + if 'pre-processor' in config: + for pre_processor_id in config['pre-processor']: + pre_processor_config = config['pre-processor'][pre_processor_id] + + if 'puller' not in pre_processor_config: + logging.error("no puller name found for pre-processor " + pre_processor_id) + raise MissingArgumentException(argument_name='puller') + + puller_id = pre_processor_config['puller'] + + if puller_id not in config['input']: + logging.error("puller actor " + puller_id + " does not exist") + raise UnexistingActorException(actor=puller_id) + + elif 'post-processor' in config: + for post_processor_id in config['post-processor']: + post_processor_config = config['post-processor'][post_processor_id] + + if 'pusher' not in post_processor_config: + logging.error("no pusher name found for post-processor " + post_processor_id) + raise MissingArgumentException(argument_name='pusher') + + pusher_id = post_processor_config['pusher'] + + if pusher_id not in config['output']: + logging.error("pusher actor " + pusher_id + " does not exist") + raise UnexistingActorException(actor=pusher_id) + ConfigValidator._validate_input(config) @staticmethod def _validate_input(config: Dict): + """ + Check that csv input type has files that exist + """ for key, input_config in config['input'].items(): if input_config['type'] == 'csv': list_of_files = input_config['files'] @@ -94,3 +128,32 @@ def _validate_input(config: Dict): for file_name in list_of_files: if not os.access(file_name, os.R_OK): raise FileDoesNotExistException(file_name=file_name) + + @staticmethod + def _validate_binding(config: Dict): + """ + Check that defined bindings use existing actors defined by the configuration + """ + for _, binding_infos in config['binding'].items(): + + if 'from' not in binding_infos: + logging.error("no from parameter found for binding") + raise MissingArgumentException(argument_name='from') + + if 'to' not in binding_infos: + logging.error("no to parameter found for binding") + raise MissingArgumentException(argument_name='to') + + # from_info[0] is the subgroup and from_info[1] the actor name + from_infos = binding_infos['from'].split('.') + + if from_infos[0] not in config or from_infos[1] not in config[from_infos[0]]: + logging.error("from actor does not exist") + raise UnexistingActorException(actor=binding_infos['from']) + + # to_info[0] is the subgroup and to_info[1] the actor name + to_infos = binding_infos['to'].split('.') + + if to_infos[0] not in config or to_infos[1] not in config[to_infos[0]]: + logging.error("to actor does not exist") + raise UnexistingActorException(actor=binding_infos['to']) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index 3102fe45..020f9f6b 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -30,20 +30,23 @@ import logging import os import sys -from typing import Dict, Type +from typing import Dict, Type, Callable from powerapi.actor import Actor from powerapi.database.influxdb2 import InfluxDB2 from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ - DatabaseNameAlreadyUsed + DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed, MonitorTypeDoesNotExist from powerapi.filter import Filter +from powerapi.processor.pre.k8s.k8s_monitor import K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ + TIMEOUT_QUERY_DEFAULT_VALUE +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor from powerapi.report import HWPCReport, PowerReport, ControlReport, ProcfsReport, Report, FormulaReport from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, DirectPrometheusDB, \ VirtioFSDB, FileDB from powerapi.puller import PullerActor from powerapi.pusher import PusherActor -from powerapi.report_modifier.libvirt_mapper import LibvirtMapper from powerapi.puller.simple.simple_puller_actor import SimplePullerActor from powerapi.pusher.simple.simple_pusher_actor import SimplePusherActor @@ -52,11 +55,29 @@ COMPONENT_DB_NAME_KEY = 'db' COMPONENT_DB_COLLECTION_KEY = 'collection' COMPONENT_DB_MANAGER_KEY = 'db_manager' -COMPONENT_DB_MAX_BUFFER_SIZE = 'max_buffer_size' +COMPONENT_DB_MAX_BUFFER_SIZE_KEY = 'max_buffer_size' +COMPONENT_URI_KEY = 'uri' + +ACTOR_NAME_KEY = 'actor_name' +TARGET_ACTORS_KEY = 'target_actors' +REGEXP_KEY = 'regexp' +K8S_API_MODE_KEY = 'k8s-api-mode' +TIME_INTERVAL_KEY = 'time-interval' +TIMEOUT_QUERY_KEY = 'timeout-query' +PULLER_NAME_KEY = 'puller' +PUSHER_NAME_KEY = 'pusher' +API_KEY_KEY = 'api-key' +HOST_KEY = 'host' + +LISTENER_ACTOR_KEY = 'listener_actor' GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' +MONITOR_NAME_SUFFIX = '_monitor' +MONITOR_KEY = 'monitor' +K8S_COMPONENT_TYPE_VALUE = 'k8s' + class Generator: """ @@ -248,18 +269,13 @@ class PullerGenerator(DBActorGenerator): Generate Puller Actor class and Puller start message from config """ - def __init__(self, report_filter: Filter, report_modifier_list=None): + def __init__(self, report_filter: Filter): DBActorGenerator.__init__(self, 'input') self.report_filter = report_filter - if report_modifier_list is None: - report_modifier_list = [] - self.report_modifier_list = report_modifier_list - def _actor_factory(self, actor_name: str, main_config, component_config: dict): return PullerActor(name=actor_name, database=component_config[COMPONENT_DB_MANAGER_KEY], report_filter=self.report_filter, stream_mode=main_config[GENERAL_CONF_STREAM_MODE_KEY], - report_modifier_list=self.report_modifier_list, report_model=component_config[COMPONENT_MODEL_KEY], level_logger=logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO) @@ -301,7 +317,7 @@ def _actor_factory(self, actor_name: str, main_config: dict, component_config: d if 'max_buffer_size' in component_config.keys(): return PusherActor(name=actor_name, report_model=component_config[COMPONENT_MODEL_KEY], database=component_config[COMPONENT_DB_MANAGER_KEY], - max_size=component_config[COMPONENT_DB_MAX_BUFFER_SIZE]) + max_size=component_config[COMPONENT_DB_MAX_BUFFER_SIZE_KEY]) return PusherActor(name=actor_name, report_model=component_config[COMPONENT_MODEL_KEY], database=component_config[COMPONENT_DB_MANAGER_KEY], @@ -321,22 +337,141 @@ def _actor_factory(self, actor_name: str, main_config: dict, component_config: d number_of_reports_to_store=component_config['number_of_reports_to_store']) -class ReportModifierGenerator: +class ProcessorGenerator(Generator): """ - Generate Report modifier list from config + Generator that initialises the processor from config + """ + + def __init__(self, component_group_name: str): + Generator.__init__(self, component_group_name=component_group_name) + + self.processor_factory = self._get_default_processor_factories() + + def _get_default_processor_factories(self) -> dict: + """ + Init the factories for this processor generator + """ + raise NotImplementedError + + def remove_processor_factory(self, processor_type: str): + """ + remove a processor from generator + """ + if processor_type not in self.processor_factory: + raise ProcessorTypeDoesNotExist(processor_type=processor_type) + del self.processor_factory[processor_type] + + def add_processor_factory(self, processor_type: str, processor_factory_function: Callable): + """ + add a processor to generator + """ + if processor_type in self.processor_factory: + raise ProcessorTypeAlreadyUsed(processor_type=processor_type) + self.processor_factory[processor_type] = processor_factory_function + + def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str): + + processor_actor_type = component_config[COMPONENT_TYPE_KEY] + + if processor_actor_type not in self.processor_factory: + msg = 'Configuration error : processor actor type ' + processor_actor_type + ' unknown' + print(msg, file=sys.stderr) + raise PowerAPIException(msg) + else: + component_config[ACTOR_NAME_KEY] = actor_name + component_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] + return self.processor_factory[processor_actor_type](component_config) + + +class PreProcessorGenerator(ProcessorGenerator): + """ + Generator that initialises the pre-processor from config + """ + + def __init__(self): + ProcessorGenerator.__init__(self, component_group_name='pre-processor') + + def _get_default_processor_factories(self) -> dict: + return { + 'libvirt': lambda processor_config: LibvirtPreProcessorActor(name=processor_config[ACTOR_NAME_KEY], + uri=processor_config[COMPONENT_URI_KEY], + regexp=processor_config[REGEXP_KEY], + target_actors_names=[processor_config + [PULLER_NAME_KEY]]), + 'k8s': lambda processor_config: K8sPreProcessorActor(name=processor_config[ACTOR_NAME_KEY], + ks8_api_mode=None if + K8S_API_MODE_KEY not in processor_config else + processor_config[K8S_API_MODE_KEY], + time_interval=TIME_INTERVAL_DEFAULT_VALUE if + TIME_INTERVAL_KEY not in processor_config else + processor_config[TIME_INTERVAL_KEY], + timeout_query=TIMEOUT_QUERY_DEFAULT_VALUE if + TIMEOUT_QUERY_KEY not in processor_config + else processor_config[TIMEOUT_QUERY_KEY], + api_key=None if API_KEY_KEY not in processor_config + else processor_config[API_KEY_KEY], + host=None if HOST_KEY not in processor_config + else processor_config[HOST_KEY], + level_logger=logging.DEBUG if + processor_config[GENERAL_CONF_VERBOSE_KEY] else + logging.INFO, + target_actors_names=[processor_config[PULLER_NAME_KEY]] + ) + } + + +class PostProcessorGenerator(ProcessorGenerator): + """ + Generator that initialises the post-processor from config """ def __init__(self): - self.factory = {'libvirt_mapper': lambda config: LibvirtMapper(config['uri'], config['domain_regexp'])} + ProcessorGenerator.__init__(self, component_group_name='pre-processor') + + def _get_default_processor_factories(self) -> dict: + return {} - def generate(self, config: dict): + +class MonitorGenerator(Generator): + """ + Generator that initialises the monitor by using a K8sPreProcessorActor + """ + + def __init__(self): + Generator.__init__(self, component_group_name=MONITOR_KEY) + + self.monitor_factory = { + K8S_COMPONENT_TYPE_VALUE: lambda monitor_config: K8sMonitorAgent( + name=monitor_config[ACTOR_NAME_KEY], + concerned_actor_state=monitor_config[LISTENER_ACTOR_KEY].state, + level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel() + ) + + } + + def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str): + + monitor_actor_type = component_config[COMPONENT_TYPE_KEY] + + if monitor_actor_type not in self.monitor_factory: + raise MonitorTypeDoesNotExist(monitor_type=monitor_actor_type) + else: + component_config[ACTOR_NAME_KEY] = actor_name + MONITOR_NAME_SUFFIX + return self.monitor_factory[monitor_actor_type](component_config) + + def generate_from_processors(self, processors: dict) -> dict: """ - Generate Report modifier list from config + Generates monitors associated with the given processors + :param dict processors: Dictionary with the processors for the generation """ - report_modifier_list = [] - if 'report_modifier' not in config: - return [] - for report_modifier_name in config['report_modifier'].keys(): - report_modifier = self.factory[report_modifier_name](config['report_modifier'][report_modifier_name]) - report_modifier_list.append(report_modifier) - return report_modifier_list + + monitors_config = {MONITOR_KEY: {}} + + for processor_name, processor in processors.items(): + + if isinstance(processor, K8sPreProcessorActor): + monitors_config[MONITOR_KEY][processor_name + MONITOR_NAME_SUFFIX] = { + COMPONENT_TYPE_KEY: K8S_COMPONENT_TYPE_VALUE, + LISTENER_ACTOR_KEY: processor} + + return self.generate(main_config=monitors_config) diff --git a/powerapi/dispatcher/dispatcher_actor.py b/powerapi/dispatcher/dispatcher_actor.py index 8443fa68..fcca6b03 100644 --- a/powerapi/dispatcher/dispatcher_actor.py +++ b/powerapi/dispatcher/dispatcher_actor.py @@ -140,6 +140,13 @@ def get_all_formula(self): """ return self.formula_dict.items() + def set_formula_factory(self, formula_factory: Callable): + """ + Set the formula_factory function + :param Callable formula_factory: The new formula_factory + """ + self.formula_factory = formula_factory + class DispatcherActor(Actor): """ @@ -164,6 +171,8 @@ def __init__(self, name: str, formula_init_function: Callable, pushers: [], rout # (func): Function for creating Formula self.formula_init_function = formula_init_function + self.pushers = pushers + # (powerapi.DispatcherState): Actor state self.state = DispatcherState(self, self._create_factory(pushers), route_table) @@ -195,3 +204,9 @@ def factory(formula_id): return formula return factory + + def update_state_formula_factory(self): + """ + Update the formula_factory function of the state by using the pusher list + """ + self.state.set_formula_factory(self._create_factory(self.pushers)) diff --git a/powerapi/dispatcher/simple/simple_dispatcher_handlers.py b/powerapi/dispatcher/simple/simple_dispatcher_handlers.py index 0ff371e2..a9feceb8 100644 --- a/powerapi/dispatcher/simple/simple_dispatcher_handlers.py +++ b/powerapi/dispatcher/simple/simple_dispatcher_handlers.py @@ -28,7 +28,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from powerapi.actor import State from powerapi.handler import StartHandler, PoisonPillMessageHandler, Handler -from powerapi.message import StartMessage, UnknownMessageTypeException +from powerapi.message import StartMessage +from powerapi.exception import UnknownMessageTypeException from powerapi.report import Report diff --git a/powerapi/exception.py b/powerapi/exception.py index 3a123675..ef5449a9 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -269,7 +269,7 @@ def __init__(self, model_name: str): class InvalidPrefixException(PowerAPIException): """ - Exception raised when attempting to add a new prefix that is a prefix of a existing one or + Exception raised when attempting to add a new prefix that is a prefix of an existing one or vice-versa """ @@ -279,3 +279,90 @@ def __init__(self, existing_prefix: str, new_prefix: str): self.existing_prefix = existing_prefix self.msg = "The new prefix " + self.new_prefix + " has a conflict with the existing prefix " \ + self.existing_prefix + + +class LibvirtException(PowerAPIException): + """ + Exception raised when there are issues regarding the import of LibvirtException + """ + + def __init__(self, _): + PowerAPIException.__init__(self) + + +class ProcessorTypeDoesNotExist(PowerAPIException): + """ + Exception raised when attempting to remove to a ProcessorActorGenerator a processor factory with a type that is not + bound to a processor factory + """ + + def __init__(self, processor_type: str): + PowerAPIException.__init__(self) + self.processor_type = processor_type + + +class ProcessorTypeAlreadyUsed(PowerAPIException): + """ + Exception raised when attempting to add to a ProcessorActorGenerator a processor factory with a type already bound + to another processor factory + """ + + def __init__(self, processor_type: str): + PowerAPIException.__init__(self) + self.processor_type = processor_type + + +class UnsupportedActorTypeException(ParserException): + """ + Exception raised when the binding manager do not support an actor type + """ + + def __init__(self, actor_type: str): + ParserException.__init__(self, argument_name=actor_type) + self.msg = 'Unsupported Actor Type ' + actor_type + + +class UnknownMessageTypeException(PowerAPIException): + """ + Exception happen when we don't know the message type + """ + + +class MonitorTypeDoesNotExist(PowerAPIException): + """ + Exception raised when attempting to remove to a MonitorGenerator a monitor factory with a type that is not + bound to a monitor factory + """ + + def __init__(self, monitor_type: str): + PowerAPIException.__init__(self) + self.monitor_type = monitor_type + + +class UnexistingActorException(PowerAPIException): + """ + Exception raised when an actor referenced in a processor does not exist + """ + + def __init__(self, actor: str): + PowerAPIException.__init__(self) + self.actor = actor + + +class BindingWrongActorsException(PowerAPIException): + """ + Exception raised when at least one of the actors in a binding is not of a given type + """ + + def __init__(self): + PowerAPIException.__init__(self) + + +class TargetActorAlreadyUsed(PowerAPIException): + """ + Exception raised when an actor is used by more than one processor + """ + + def __init__(self, target_actor: str): + PowerAPIException.__init__(self) + self.target_actor = target_actor diff --git a/powerapi/handler/handler.py b/powerapi/handler/handler.py index 47e94e9e..c0e861d5 100644 --- a/powerapi/handler/handler.py +++ b/powerapi/handler/handler.py @@ -27,8 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.exception import PowerAPIException -from powerapi.message import Message, UnknownMessageTypeException +from powerapi.exception import PowerAPIException, UnknownMessageTypeException +from powerapi.message import Message class HandlerException(PowerAPIException): diff --git a/powerapi/handler/poison_pill_message_handler.py b/powerapi/handler/poison_pill_message_handler.py index 2e5c657f..62395763 100644 --- a/powerapi/handler/poison_pill_message_handler.py +++ b/powerapi/handler/poison_pill_message_handler.py @@ -27,7 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.message import UnknownMessageTypeException, PoisonPillMessage +from powerapi.message import PoisonPillMessage +from powerapi.exception import UnknownMessageTypeException from .handler import Handler diff --git a/powerapi/message.py b/powerapi/message.py index 9ccaef1b..cd61897f 100644 --- a/powerapi/message.py +++ b/powerapi/message.py @@ -30,14 +30,12 @@ from __future__ import annotations from typing import TYPE_CHECKING -from powerapi.exception import PowerAPIException if TYPE_CHECKING: from powerapi.database import BaseDB from powerapi.filter import Filter from powerapi.dispatcher import RouteTable from powerapi.formula import FormulaActor, FormulaState - from powerapi.report_modifier import ReportModifier class Message: @@ -170,9 +168,3 @@ def __eq__(self, other): if isinstance(other, PoisonPillMessage): return other.is_soft == self.is_soft and other.is_hard == self.is_hard return False - - -class UnknownMessageTypeException(PowerAPIException): - """ - Exception happen when we don't know the message type - """ diff --git a/powerapi/report_modifier/__init__.py b/powerapi/processor/__init__.py similarity index 87% rename from powerapi/report_modifier/__init__.py rename to powerapi/processor/__init__.py index e9faee94..f964fff4 100644 --- a/powerapi/report_modifier/__init__.py +++ b/powerapi/processor/__init__.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -15,7 +15,7 @@ # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -26,6 +26,3 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.report_modifier.report_modifier import ReportModifier -from powerapi.report_modifier.libvirt_mapper import LibvirtMapper diff --git a/tests/unit/report_modifier/test_libvirt_mapper.py b/powerapi/processor/handlers.py similarity index 53% rename from tests/unit/report_modifier/test_libvirt_mapper.py rename to powerapi/processor/handlers.py index 9d02aac1..03a23c36 100644 --- a/tests/unit/report_modifier/test_libvirt_mapper.py +++ b/powerapi/processor/handlers.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -15,7 +15,7 @@ # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -26,43 +26,18 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from mock import patch - -try: - from libvirt import libvirtError -except ImportError: - libvirtError = Exception - -from powerapi.report_modifier import LibvirtMapper +from powerapi.handler import InitHandler from powerapi.report import Report -from tests.utils.libvirt import MockedLibvirt, LIBVIRT_TARGET_NAME1, LIBVIRT_TARGET_NAME2, UUID_1, REGEXP - -BAD_TARGET = 'lkjqlskjdlqksjdlkj' - - -@pytest.fixture -def libvirt_mapper(): - with patch('powerapi.report_modifier.libvirt_mapper.openReadOnly', return_value=MockedLibvirt()): - return LibvirtMapper('', REGEXP) - - -def test_modify_report_that_not_match_regexp_musnt_modify_report(libvirt_mapper): - report = Report(0, 'sensor', BAD_TARGET) - new_report = libvirt_mapper.modify_report(report) - assert new_report.target == BAD_TARGET - assert new_report.metadata == {} - - -def test_modify_report_that_match_regexp_must_modify_report(libvirt_mapper): - report = Report(0, 'sensor', LIBVIRT_TARGET_NAME1) - new_report = libvirt_mapper.modify_report(report) - assert new_report.metadata["domain_id"] == UUID_1 +class ProcessorReportHandler(InitHandler): + """ + Processor the report by modifying it in some way and then send the modified report to targets actos + """ -def test_modify_report_that_match_regexp_but_with_wrong_domain_name_musnt_modify_report(libvirt_mapper): - report = Report(0, 'sensor', LIBVIRT_TARGET_NAME2) - new_report = libvirt_mapper.modify_report(report) - assert new_report.metadata == {} + def _send_report(self, report: Report): + """ + Send the report to the actor target + """ + for target in self.state.target_actors: + target.send_data(report) diff --git a/powerapi/report_modifier/report_modifier.py b/powerapi/processor/pre/__init__.py similarity index 79% rename from powerapi/report_modifier/report_modifier.py rename to powerapi/processor/pre/__init__.py index 9a441327..f964fff4 100644 --- a/powerapi/report_modifier/report_modifier.py +++ b/powerapi/processor/pre/__init__.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -15,7 +15,7 @@ # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -26,16 +26,3 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.report import Report - - -class ReportModifier: - """ - Abstract class for object used by puller to modify reports before sending them to dispatcher - """ - def modify_report(self, report: Report) -> Report: - """ - modify the given report - """ - raise NotImplementedError() diff --git a/powerapi/processor/pre/k8s/__init__.py b/powerapi/processor/pre/k8s/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/pre/k8s/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py new file mode 100644 index 00000000..affc1929 --- /dev/null +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -0,0 +1,254 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=W0603,W0718 + +from logging import Formatter, getLogger, Logger, StreamHandler, WARNING +from multiprocessing import Manager, Process + +from kubernetes import client, config, watch +from kubernetes.client.configuration import Configuration +from kubernetes.client.rest import ApiException + +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorState, K8sMetadataCacheManager, \ + K8sPodUpdateMetadata, DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT + +LOCAL_CONFIG_MODE = "local" +MANUAL_CONFIG_MODE = "manual" +CLUSTER_CONFIG_MODE = "cluster" + +MANUAL_CONFIG_API_KEY_DEFAULT_VALUE = "YOUR_API_KEY" +MANUAL_CONFIG_HOST_DEFAULT_VALUE = "http://localhost" + +v1_api = None + +manual_config_api_key = MANUAL_CONFIG_API_KEY_DEFAULT_VALUE +manual_config_host = MANUAL_CONFIG_HOST_DEFAULT_VALUE + + +def local_config(): + """ + Return local kubectl + """ + config.load_kube_config() + + +def manual_config(): + """ + Return the manual configuration + """ + # Manual config + configuration = client.Configuration() + # Configure API key authorization: BearerToken + configuration.api_key["authorization"] = manual_config_api_key + # Defining host is optional and default to http://localhost + configuration.host = manual_config_host + Configuration.set_default(configuration) + + +def cluster_config(): + """ + Return the cluster configuration + """ + config.load_incluster_config() + + +def load_k8s_client_config(logger: Logger, mode: str = None): + """ + Load K8S client configuration according to the `mode`. + If no mode is given `LOCAL_CONFIG_MODE` is used. + params: + mode : one of `LOCAL_CONFIG_MODE`, `MANUAL_CONFIG_MODE` + or `CLUSTER_CONFIG_MODE` + """ + logger.debug("Loading k8s api conf mode %s ", mode) + { + LOCAL_CONFIG_MODE: local_config, + MANUAL_CONFIG_MODE: manual_config, + CLUSTER_CONFIG_MODE: cluster_config, + }.get(mode, local_config)() + + +def get_core_v1_api(logger: Logger, mode: str = None, api_key: str = None, host: str = None): + """ + Returns a handler to the k8s API. + """ + global v1_api + global manual_config_api_key + global manual_config_host + if v1_api is None: + if api_key: + manual_config_api_key = api_key + if host: + manual_config_host = host + load_k8s_client_config(logger=logger, mode=mode) + v1_api = client.CoreV1Api() + logger.info(f"Core v1 api access : {v1_api}") + return v1_api + + +def extract_containers(pod_obj): + """ + Extract the containers ids from a pod + :param pod_obj: Pod object for extracting the containers ids + """ + if not pod_obj.status.container_statuses: + return [] + + container_ids = [] + for container_status in pod_obj.status.container_statuses: + container_id = container_status.container_id + if not container_id: + continue + # container_id actually depends on the container engine used by k8s. + # It seems that is always start with :// + # e.g. + # 'containerd://2289b494f36b93647cfefc6f6ed4d7f36161d5c2f92d1f23571878a4e85282ed' + container_id = container_id[container_id.index("//") + 2:] + container_ids.append(container_id) + + return sorted(container_ids) + + +class K8sMonitorAgent(Process): + """ + A monitors the k8s API and sends messages + when pod are created, removed or modified. + """ + + def __init__(self, name: str, concerned_actor_state: K8sPreProcessorState, level_logger: int = WARNING): + """ + :param str name: The actor name + :param K8sPreProcessorState concerned_actor_state: state of the actor that will use the monitored information + :pram int level_logger: The logger level + """ + Process.__init__(self, name=name) + + #: (logging.Logger): Logger + self.logger = getLogger(name) + self.logger.setLevel(level_logger) + formatter = Formatter('%(asctime)s || %(levelname)s || ' + '%(process)d %(processName)s || %(message)s') + handler = StreamHandler() + handler.setFormatter(formatter) + + # Concerned Actor state + self.concerned_actor_state = concerned_actor_state + + # Multiprocessing Manager + self.manager = Manager() + + # Shared cache + self.concerned_actor_state.metadata_cache_manager = K8sMetadataCacheManager(process_manager=self.manager, + level_logger=level_logger) + + self.stop_monitoring = self.manager.Event() + + self.stop_monitoring.clear() + + def run(self): + """ + Main code executed by the Monitor + """ + self.query_k8s() + + def query_k8s(self): + """ + Query k8s for changes and update the metadata cache + """ + try: + while not self.stop_monitoring.is_set(): + events = self.k8s_streaming_query() + for event in events: + event_type, namespace, pod_name, container_ids, labels = event + + self.concerned_actor_state.metadata_cache_manager.update_cache(metadata=K8sPodUpdateMetadata( + event=event_type, + namespace=namespace, + pod=pod_name, + containers_id=container_ids, + labels=labels + ) + ) + + self.stop_monitoring.wait(timeout=self.concerned_actor_state.time_interval) + + except BrokenPipeError: + # This error can happen when stopping the monitor process + return + except Exception as ex: + self.logger.warning("Failed streaming query %s", ex) + finally: + self.manager.shutdown() + + def k8s_streaming_query(self) -> list: + """ + Return a list of events by using the provided parameters + :param int timeout_seconds: Timeout in seconds for waiting for events + :param str k8sapi_mode: Kind of API mode + """ + api = get_core_v1_api(mode=self.concerned_actor_state.k8s_api_mode, logger=self.logger, + api_key=self.concerned_actor_state.api_key, host=self.concerned_actor_state.host) + events = [] + w = watch.Watch() + + try: + event = None + for event in w.stream( + func=api.list_pod_for_all_namespaces, timeout_seconds=self.concerned_actor_state.timeout_query + ): + + if event: + + if event["type"] not in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: + self.logger.warning( + "UNKNOWN EVENT TYPE : %s : %s %s", + event['type'], event['object'].metadata.name, event + ) + continue + + pod_obj = event["object"] + + namespace, pod_name = \ + pod_obj.metadata.namespace, pod_obj.metadata.name + + container_ids = ( + [] if event["type"] == "DELETED" + else extract_containers(pod_obj) + ) + + labels = pod_obj.metadata.labels + events.append( + (event["type"], namespace, pod_name, container_ids, labels) + ) + + except ApiException as ae: + self.logger.error("APIException %s %s", ae.status, ae) + except Exception as undef_e: + self.logger.error("Error when watching Exception %s %s", undef_e, event) + return events diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py new file mode 100644 index 00000000..841907fc --- /dev/null +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py @@ -0,0 +1,216 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +This module provides two k8s specific actors +* `K8sProcessorActor`, which add k8s metadata to reports and forward them to another actor. +* `K8sMonitorAgent`, which monitors the k8s API and sends messages when pod are created, removed or modified. +""" +import logging +from multiprocessing.managers import BaseManager + +from typing import Tuple, Dict, List + +from powerapi.actor import Actor +from powerapi.message import StartMessage, PoisonPillMessage +from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import K8sPreProcessorActorHWPCReportHandler, \ + K8sPreProcessorActorStartMessageHandler, K8sPreProcessorActorPoisonPillMessageHandler +from powerapi.processor.processor_actor import ProcessorState, ProcessorActor +from powerapi.report import HWPCReport + +ADDED_EVENT = 'ADDED' +DELETED_EVENT = 'DELETED' +MODIFIED_EVENT = 'MODIFIED' + +DEFAULT_K8S_CACHE_MANAGER_NAME = 'k8s_cache_manager' +DEFAULT_K8S_MONITOR_NAME = 'k8s_monitor' + +POD_LABELS_KEY = 'pod_labels' +CONTAINERS_POD_KEY = 'containers_pod' +POD_CONTAINERS_KEY = 'pod_containers' + +TIME_INTERVAL_DEFAULT_VALUE = 0 +TIMEOUT_QUERY_DEFAULT_VALUE = 5 + + +class K8sPodUpdateMetadata: + """ + Metadata related to a monitored event + """ + + def __init__( + self, + event: str, + namespace: str, + pod: str, + containers_id: List[str] = None, + labels: Dict[str, str] = None, + ): + """ + :param str event: Event name + :param str namespace: Namespace name + :param str pod: Id of the Pod + :param list containers_id: List of containers id + :param dict labels: Dictionary of labels + """ + self.event = event + self.namespace = namespace + self.pod = pod + self.containers_id = containers_id if containers_id is not None else [] + self.labels = labels if labels is not None else {} + + def __str__(self): + return f"K8sPodUpdateMetadata {self.event} {self.namespace} {self.pod}" + + def __eq__(self, metadata): + return (self.event, self.namespace, self.pod, self.containers_id, self.labels) == \ + (metadata.event, metadata.namespace, metadata.pod, metadata.containers_id, metadata.labels) + + +class K8sMetadataCacheManager: + """ + K8sMetadataCache maintains a cache of pods' metadata + (namespace, labels and id of associated containers) + """ + + def __init__(self, process_manager: BaseManager, name: str = DEFAULT_K8S_CACHE_MANAGER_NAME, + level_logger: int = logging.WARNING): + + # Dictionaries + + self.process_manager = process_manager + + self.pod_labels = self.process_manager.dict() # (ns, pod) => [labels] + + self.containers_pod = self.process_manager.dict() # container_id => (ns, pod) + + self.pod_containers = self.process_manager.dict() # (ns, pod) => [container_ids] + + # Logger + self.logger = logging.getLogger(name) + self.logger.setLevel(level_logger) + + def update_cache(self, metadata: K8sPodUpdateMetadata): + """ + Update the local cache for pods. + + Register this function as a callback for K8sMonitorAgent messages. + """ + if metadata.event == ADDED_EVENT: + self.pod_labels[(metadata.namespace, metadata.pod)] = metadata.labels + self.pod_containers[(metadata.namespace, metadata.pod)] = metadata.containers_id + for container_id in metadata.containers_id: + self.containers_pod[container_id] = \ + (metadata.namespace, metadata.pod) + + elif metadata.event == DELETED_EVENT: + self.pod_labels.pop((metadata.namespace, metadata.pod), None) + for container_id in self.pod_containers.pop((metadata.namespace, metadata.pod), []): + self.containers_pod.pop(container_id, None) + # logger.debug("Pod removed %s %s", message.namespace, message.pod) + + elif metadata.event == MODIFIED_EVENT: + self.pod_labels[(metadata.namespace, metadata.pod)] = metadata.labels + for prev_container_id in self.pod_containers.pop((metadata.namespace, metadata.pod), []): + self.pod_containers.pop(prev_container_id, None) + self.pod_containers[(metadata.namespace, metadata.pod)] = metadata.containers_id + for container_id in metadata.containers_id: + self.containers_pod[container_id] = (metadata.namespace, metadata.pod) + + else: + self.logger.error("Error : unknown event type %s ", metadata.event) + + def get_container_pod(self, container_id: str) -> Tuple[str, str]: + """ + Get the pod for a container_id. + + :param str container_id: Id of the container + :return a tuple (namespace, pod_name) of (None, None) if no pod + could be found for this container + """ + ns_pod = self.containers_pod.get(container_id, None) + if ns_pod is None: + return None, None + return ns_pod + + def get_pod_labels(self, namespace: str, pod_name: str) -> Dict[str, str]: + """ + Get labels for a pod. + + :param str namespace: The namespace related to the pod + :param str pod_name: The name of the pod + :return a dict {label_name, label_value} + """ + return self.pod_labels.get((namespace, pod_name), dict) + + +class K8sPreProcessorState(ProcessorState): + """ + State related to a K8SProcessorActor + """ + + def __init__(self, actor: Actor, target_actors: list, + target_actors_names: list, k8s_api_mode: str, time_interval: int, timeout_query: int, api_key: str, + host: str): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) + self.metadata_cache_manager = None + self.k8s_api_mode = k8s_api_mode + self.time_interval = time_interval + self.timeout_query = timeout_query + self.api_key = api_key + self.host = host + + +class K8sPreProcessorActor(ProcessorActor): + """ + Pre-processor Actor that modifies reports by adding K8s related metadata + """ + + def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, target_actors_names: list = None, + level_logger: int = logging.WARNING, + timeout: int = 5000, time_interval: int = TIME_INTERVAL_DEFAULT_VALUE, + timeout_query: int = TIMEOUT_QUERY_DEFAULT_VALUE, api_key: str = None, host: str = None): + ProcessorActor.__init__(self, name=name, level_logger=level_logger, + timeout=timeout) + + self.state = K8sPreProcessorState(actor=self, + target_actors=target_actors, + k8s_api_mode=ks8_api_mode, time_interval=time_interval, + timeout_query=timeout_query, target_actors_names=target_actors_names, + api_key=api_key, host=host) + + def setup(self): + """ + Define HWPCReportMessage handler, StartMessage handler and PoisonPillMessage Handler + """ + ProcessorActor.setup(self) + self.add_handler(message_type=StartMessage, handler=K8sPreProcessorActorStartMessageHandler(state=self.state)) + self.add_handler(message_type=HWPCReport, handler=K8sPreProcessorActorHWPCReportHandler(state=self.state)) + self.add_handler(message_type=PoisonPillMessage, + handler=K8sPreProcessorActorPoisonPillMessageHandler(state=self.state)) diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py new file mode 100644 index 00000000..16aca2a2 --- /dev/null +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -0,0 +1,123 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from powerapi.actor import State +from powerapi.handler import StartHandler, PoisonPillMessageHandler +from powerapi.message import Message +from powerapi.processor.handlers import ProcessorReportHandler + +POD_NAMESPACE_METADATA_KEY = 'pod_namespace' +POD_NAME_METADATA_KEY = 'pod_name' + + +class K8sPreProcessorActorStartMessageHandler(StartHandler): + """ + Start the K8sProcessorActor + """ + + def __init__(self, state: State): + StartHandler.__init__(self, state=state) + + # + def initialization(self): + for actor in self.state.target_actors: + actor.connect_data() + + +class K8sPreProcessorActorHWPCReportHandler(ProcessorReportHandler): + """ + Process the HWPC Reports + """ + + def __init__(self, state: State): + ProcessorReportHandler.__init__(self, state=state) + + def handle(self, message: Message): + + # Add pod name, namespace and labels to the report + c_id = clean_up_container_id(message.target) + + namespace, pod = self.state.metadata_cache_manager.get_container_pod(c_id) + if namespace is None or pod is None: + self.state.actor.logger.warning( + f"Container with no associated pod : {message.target}, {c_id}, {namespace}, {pod}" + ) + else: + message.metadata[POD_NAMESPACE_METADATA_KEY] = namespace + message.metadata[POD_NAME_METADATA_KEY] = pod + self.state.actor.logger.debug( + f"K8sPreProcessorActorHWPCReportHandler add metadata to report {c_id}, {namespace}, {pod}" + ) + + labels = self.state.metadata_cache_manager.get_pod_labels(namespace, pod) + for label_key, label_value in labels.items(): + message.metadata[f"label_{label_key}"] = label_value + + self._send_report(report=message) + + +class K8sPreProcessorActorPoisonPillMessageHandler(PoisonPillMessageHandler): + """ + Stop the K8sProcessorActor + """ + + def __init__(self, state: State): + PoisonPillMessageHandler.__init__(self, state=state) + + def teardown(self, soft=False): + for actor in self.state.target_actors: + actor.close() + + +def clean_up_container_id(c_id): + """ + On some system, we receive a container id that requires some cleanup to match + the id returned by the k8s api + k8s creates cgroup directories, which is what we get as id from the sensor, + according to this pattern: + /kubepods//pod/ + depending on the container engine, we need to clean up the part + """ + + if "/docker-" in c_id: + # for path like : + # /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod435532e3_546d_45e2_8862_d3c7b320d2d9.slice/ + # docker-68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a.scope + # where we actually only want the end of that path : + # 68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a + try: + return c_id[c_id.rindex("/docker-") + 8: -6] + except ValueError: + return c_id + else: + # /kubepods/besteffort/pod42006d2c-cad7-4575-bfa3-91848a558743/ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5 + try: + return c_id[c_id.rindex("/") + 1:] + except ValueError: + return c_id diff --git a/powerapi/processor/pre/libvirt/__init__.py b/powerapi/processor/pre/libvirt/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/pre/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py b/powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py new file mode 100644 index 00000000..892edc02 --- /dev/null +++ b/powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import re + +from powerapi.exception import LibvirtException +from powerapi.message import StartMessage +from powerapi.processor.pre.libvirt.libvirt_pre_processor_handlers import LibvirtPreProcessorReportHandler, \ + LibvirtPreProcessorStartHandler +from powerapi.report import Report +from powerapi.actor import Actor +from powerapi.processor.processor_actor import ProcessorActor, ProcessorState + +try: + from libvirt import openReadOnly +except ImportError: + logging.getLogger().info("libvirt-python is not installed.") + + libvirtError = LibvirtException + openReadOnly = None + + +class LibvirtPreProcessorState(ProcessorState): + """ + State related to a LibvirtPreProcessorActor + """ + + def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list, target_actors_names: list): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) + self.regexp = re.compile(regexp) + self.daemon_uri = None if uri == '' else uri + self.libvirt = openReadOnly(self.daemon_uri) + + +class LibvirtPreProcessorActor(ProcessorActor): + """ + Processor Actor that modifies reports by replacing libvirt id by open stak uuid + """ + + def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, target_actors_names: list = None, + level_logger: int = logging.WARNING, + timeout: int = 5000): + ProcessorActor.__init__(self, name=name, level_logger=level_logger, + timeout=timeout) + self.state = LibvirtPreProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors, + target_actors_names=target_actors_names) + + def setup(self): + """ + Define ReportMessage handler and StartMessage handler + """ + ProcessorActor.setup(self) + self.add_handler(message_type=StartMessage, handler=LibvirtPreProcessorStartHandler(state=self.state)) + self.add_handler(message_type=Report, handler=LibvirtPreProcessorReportHandler(state=self.state)) diff --git a/powerapi/report_modifier/libvirt_mapper.py b/powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py similarity index 61% rename from powerapi/report_modifier/libvirt_mapper.py rename to powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py index ff30112e..f33cb963 100644 --- a/powerapi/report_modifier/libvirt_mapper.py +++ b/powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py @@ -1,5 +1,5 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -15,7 +15,7 @@ # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -26,43 +26,57 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import re import logging +import re + +from powerapi.actor import State +from powerapi.exception import LibvirtException +from powerapi.processor.handlers import ProcessorReportHandler +from powerapi.handler import StartHandler +from powerapi.report import Report try: - from libvirt import openReadOnly, libvirtError + from libvirt import libvirtError except ImportError: logging.getLogger().info("libvirt-python is not installed.") - class LibvirtException(Exception): - """""" - def __init__(self, _): - Exception.__init__(self) - libvirtError = LibvirtException - openReadOnly = None - -from powerapi.report_modifier.report_modifier import ReportModifier -class LibvirtMapper(ReportModifier): +class LibvirtPreProcessorReportHandler(ProcessorReportHandler): """ - Report modifier which modifi target with libvirt id by open stak uuid + Modify reports by replacing libvirt id by open stak uuid """ - def __init__(self, uri: str, regexp: str): - self.regexp = re.compile(regexp) - daemon_uri = None if uri == '' else uri - self.libvirt = openReadOnly(daemon_uri) + def __init__(self, state): + ProcessorReportHandler.__init__(self, state=state) - def modify_report(self, report): - result = re.match(self.regexp, report.target) + def handle(self, report: Report): + """ + Modify reports by replacing libvirt id by open stak uuid + + :param Report report: Report to be modified + """ + result = re.match(self.state.regexp, report.target) if result is not None: domain_name = result.groups(0)[0] try: - domain = self.libvirt.lookupByName(domain_name) + domain = self.state.libvirt.lookupByName(domain_name) report.metadata["domain_id"] = domain.UUIDString() except libvirtError: pass - return report + + self._send_report(report=report) + + +class LibvirtPreProcessorStartHandler(StartHandler): + """ + Initialize the target actors + """ + + def __init__(self, state: State): + StartHandler.__init__(self, state=state) + + def initialization(self): + for actor in self.state.target_actors: + actor.connect_data() diff --git a/powerapi/processor/processor_actor.py b/powerapi/processor/processor_actor.py new file mode 100644 index 00000000..ca5fba9b --- /dev/null +++ b/powerapi/processor/processor_actor.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging + +from powerapi.actor import State, Actor +from powerapi.handler import PoisonPillMessageHandler +from powerapi.message import PoisonPillMessage + + +class ProcessorState(State): + """ + Processor Actor State + + Contains in addition to State values : + - the targets actors + """ + + def __init__(self, actor: Actor, target_actors: list, target_actors_names: list): + """ + :param list target_actors: List of target actors for the processor + """ + super().__init__(actor) + + if not target_actors: + target_actors = [] + + self.target_actors = target_actors + self.target_actors_names = target_actors_names + + +class ProcessorActor(Actor): + """ + ProcessorActor class + + A processor modifies a report and sends the modified report to a list of targets + actor. + """ + + def __init__(self, name: str, level_logger: int = logging.WARNING, timeout: int = 5000): + Actor.__init__(self, name, level_logger, timeout) + self.state = ProcessorState(actor=self, target_actors=[], target_actors_names=[]) + + def setup(self): + """ + Define PoisonPillMessage handler + """ + self.add_handler(message_type=PoisonPillMessage, handler=PoisonPillMessageHandler(state=self.state)) + + def add_target_actor(self, actor: Actor): + """ + Add the given actor to the list of targets + :param actor: Actor to be defined as target + """ + self.state.target_actors.append(actor) diff --git a/powerapi/puller/handlers.py b/powerapi/puller/handlers.py index 24817a19..5535c76f 100644 --- a/powerapi/puller/handlers.py +++ b/powerapi/puller/handlers.py @@ -87,17 +87,12 @@ def _pull_database(self): def _get_dispatchers(self, report): return self.state.report_filter.route(report) - def _modify_report(self, report): - for report_modifier in self.state.report_modifier_list: - report = report_modifier.modify_report(report) - return report - def run(self): """ Read data from Database and send it to the dispatchers. If there is no more data, send a kill message to every dispatcher. - If stream mode is disable, kill the actor. + If stream mode is disabled, kill the actor. :param None msg: None. """ @@ -113,11 +108,10 @@ def run(self): while self.state.alive: try: raw_report = self._pull_database() - report = self._modify_report(raw_report) - dispatchers = self._get_dispatchers(report) + dispatchers = self._get_dispatchers(raw_report) for dispatcher in dispatchers: - dispatcher.send_data(report) + dispatcher.send_data(raw_report) except NoReportExtractedException: time.sleep(self.state.timeout_puller / 1000) diff --git a/powerapi/puller/puller_actor.py b/powerapi/puller/puller_actor.py index 0a724319..0c2454df 100644 --- a/powerapi/puller/puller_actor.py +++ b/powerapi/puller/puller_actor.py @@ -51,7 +51,7 @@ class PullerState(State): """ def __init__(self, actor: Actor, database, report_filter, report_model, stream_mode, timeout_puller, - report_modifier_list=[], asynchrone=False): + asynchrone=False): """ :param BaseDB database: Allow to interact with a Database :param Filter report_filter: Filter of the Puller @@ -84,8 +84,6 @@ def __init__(self, actor: Actor, database, report_filter, report_model, stream_m self.loop = None - self.report_modifier_list = report_modifier_list - class PullerActor(Actor): """ @@ -95,7 +93,7 @@ class PullerActor(Actor): to many Dispatcher depending on some rules. """ - def __init__(self, name, database, report_filter, report_model, stream_mode=False, report_modifier_list=[], + def __init__(self, name, database, report_filter, report_model, stream_mode=False, level_logger=logging.WARNING, timeout=5000, timeout_puller=100): """ @@ -110,8 +108,7 @@ def __init__(self, name, database, report_filter, report_model, stream_mode=Fals Actor.__init__(self, name, level_logger, timeout) #: (State): Actor State. self.state = PullerState(self, database=database, report_filter=report_filter, report_model=report_model, - stream_mode=stream_mode, timeout_puller=timeout_puller, - report_modifier_list=report_modifier_list, asynchrone=database.asynchrone) + stream_mode=stream_mode, timeout_puller=timeout_puller, asynchrone=database.asynchrone) self.low_exception += database.exceptions diff --git a/powerapi/puller/simple/simple_puller_actor.py b/powerapi/puller/simple/simple_puller_actor.py index d25c5c3f..3c5faa13 100644 --- a/powerapi/puller/simple/simple_puller_actor.py +++ b/powerapi/puller/simple/simple_puller_actor.py @@ -39,13 +39,13 @@ class SimplePullerState(State): """ - Simple Puller Actor State + Simple Puller Actor State - Contains in addition to State values : - - the number of reports to send - - the report type to send - - the report filter - """ + Contains in addition to State values : + - the number of reports to send + - the report type to send + - the report filter + """ def __init__(self, actor, number_of_reports_to_send: int, report_type_to_send: Type[Report], report_filter): """ diff --git a/powerapi/puller/simple/simple_puller_handlers.py b/powerapi/puller/simple/simple_puller_handlers.py index 89146ffd..61dbb762 100644 --- a/powerapi/puller/simple/simple_puller_handlers.py +++ b/powerapi/puller/simple/simple_puller_handlers.py @@ -29,7 +29,8 @@ from powerapi.actor import State from powerapi.handler import Handler, StartHandler -from powerapi.message import Message, SimplePullerSendReportsMessage, UnknownMessageTypeException +from powerapi.message import Message, SimplePullerSendReportsMessage +from powerapi.exception import UnknownMessageTypeException from powerapi.puller.handlers import PullerInitializationException diff --git a/pyproject.toml b/pyproject.toml index 02f5d8a0..9f0001ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ authors = [ dependencies = [ "pyzmq >= 18.1.0", - "setproctitle >= 1.1.8", + "setproctitle >= 1.1.8" ] [project.optional-dependencies] @@ -59,10 +59,11 @@ prometheus = ["prometheus-client >= 0.9.0"] # Plaforms: libvirt = ["libvirt-python >= 6.1.0"] # requires libvirt lib/headers, do not include by default. +kubernetes = ["kubernetes >= 27.0.2"] # Aliases: all-databases = ["powerapi[mongodb, influxdb, opentsdb, prometheus]"] -all-platforms = [] +all-platforms = ["kubernetes"] everything = ["powerapi[all-databases, all-platforms]"] devel = ["powerapi[everything, test, docs, lint]"] diff --git a/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py b/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py deleted file mode 100644 index 39922ea7..00000000 --- a/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -""" -Test the behaviour of the most simple architecture with a libvirt mapper - -Architecture : - - 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off, with a report_modifier (LibvirtMapper)) - - 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET) - - 1 Dummy Formula - - 1 pusher (connected to MongoDB1 [collection test_result] - -MongoDB1 content: -- 10 HWPCReports with 2 socket for target LIBVIRT_TARGET_NAME1 -- 10 HWPCReports with 2 socket for target LIBVIRT_TARGET_NAME2 - -Scenario: - - Launch the full architecture with a libvirt mapper connected to a fake libvirt daemon which only know LIBVIRT_INSTANCE_NAME1 -Test if: - - each HWPCReport in the intput database was converted in one PowerReport per - socket in the output database - - only target name LIBVIRT_TARGET_NAME1 was converted into UUID_1 -""" -# pylint: disable=redefined-outer-name,disable=unused-argument,disable=unused-import -import time -from datetime import datetime -from mock import patch - -import pytest -import pymongo - -from powerapi.actor import Supervisor -from tests.utils.formula.dummy import DummyFormulaActor -from tests.utils.acceptation import launch_simple_architecture, SOCKET_DEPTH_LEVEL, LIBVIRT_CONFIG -# noinspection PyUnresolvedReferences -from tests.utils.db.mongo import MONGO_URI, MONGO_INPUT_COLLECTION_NAME, MONGO_OUTPUT_COLLECTION_NAME, \ - MONGO_DATABASE_NAME, mongo_database -from tests.utils.report.hwpc import extract_all_events_reports_with_vm_name -from tests.utils.libvirt import MockedLibvirt, LIBVIRT_TARGET_NAME1, UUID_1 - - -@pytest.fixture -def mongodb_content(): - """ - Get reports from a file for testing purposes - """ - return extract_all_events_reports_with_vm_name(20) - - -def check_db(): - """ - Verify that output DB has correct information - """ - mongo = pymongo.MongoClient(MONGO_URI) - c_input = mongo[MONGO_DATABASE_NAME][MONGO_INPUT_COLLECTION_NAME] - c_output = mongo[MONGO_DATABASE_NAME][MONGO_OUTPUT_COLLECTION_NAME] - - assert c_output.count_documents({}) == c_input.count_documents({}) * 2 - for report in c_input.find({"target": LIBVIRT_TARGET_NAME1}): - ts = datetime.strptime(report['timestamp'], "%Y-%m-%dT%H:%M:%S.%f") - for socket_report in c_output.find({'timestamp': ts, 'sensor': report['sensor']}): - assert socket_report["metadata"]["domain_id"] == UUID_1 - - -@patch('powerapi.report_modifier.libvirt_mapper.openReadOnly', return_value=MockedLibvirt()) -def test_run(mocked_libvirt, mongo_database, shutdown_system): - """ - Check that the actor system behave correctly with libvirt mapper - """ - supervisor = Supervisor() - launch_simple_architecture(config=LIBVIRT_CONFIG, supervisor=supervisor, hwpc_depth_level=SOCKET_DEPTH_LEVEL, - formula_class=DummyFormulaActor, generate_report_modifier_list=True) - time.sleep(2) - - check_db() - - supervisor.kill_actors() diff --git a/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py b/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py index 623343e0..255a6d87 100644 --- a/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py +++ b/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py @@ -90,7 +90,7 @@ def term_handler(_, __): launch_simple_architecture(config=get_basic_config_with_stream(), supervisor=self.supervisor, hwpc_depth_level=SOCKET_DEPTH_LEVEL, - formula_class=DummyFormulaActor, generate_report_modifier_list=True) + formula_class=DummyFormulaActor) @pytest.fixture diff --git a/tests/unit/actor/abstract_test_actor.py b/tests/unit/actor/abstract_test_actor.py index 081d2366..e34db9a3 100644 --- a/tests/unit/actor/abstract_test_actor.py +++ b/tests/unit/actor/abstract_test_actor.py @@ -206,6 +206,7 @@ def handle_message(self, msg: CrashMessage): PUSHER_NAME_POWER_REPORT = 'fake_pusher_power' PUSHER_NAME_HWPC_REPORT = 'fake_pusher_hwpc' +TARGET_ACTOR_NAME = 'fake_target_actor' REPORT_TYPE_TO_BE_SENT = PowerReport REPORT_TYPE_TO_BE_SENT_2 = HWPCReport @@ -236,6 +237,13 @@ def actor(self): """ raise NotImplementedError() + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + raise NotImplementedError() + @pytest.fixture def init_actor(self, actor): actor.start() @@ -265,6 +273,18 @@ def started_fake_pusher_power_report(self, dummy_pipe_in): join_actor(pusher) + @pytest.fixture + def started_fake_target_actor(self, report_to_be_sent, dummy_pipe_in): + """ + Return a started DummyActor. When the test is finished, the actor is stopped + """ + target_actor = DummyActor(name=TARGET_ACTOR_NAME, pipe=dummy_pipe_in, message_type=report_to_be_sent) + target_actor.start() + + yield target_actor + if target_actor.is_alive(): + target_actor.terminate() + @pytest.fixture def started_fake_pusher_hwpc_report(self, dummy_pipe_in): pusher = DummyActor(PUSHER_NAME_HWPC_REPORT, dummy_pipe_in, REPORT_TYPE_TO_BE_SENT_2) @@ -316,8 +336,7 @@ def test_send_PoisonPillMessage_set_actor_alive_to_False(self, init_actor): def test_send_StartMessage_answer_OkMessage(self, init_actor): init_actor.send_control(StartMessage(SENDER_NAME)) msg = init_actor.receive_control(2000) - print('Message....') - print(msg) + print('message start', str(msg)) assert isinstance(msg, OKMessage) def test_send_StartMessage_to_already_started_actor_answer_ErrorMessage(self, started_actor): diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 53dc8612..2ac51a52 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -31,6 +31,11 @@ import pytest import tests.utils.cli as test_files_module +from powerapi.cli.binding_manager import PreProcessorBindingManager +from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator, COMPONENT_TYPE_KEY, \ + LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX, PreProcessorGenerator +from powerapi.dispatcher import DispatcherActor, RouteTable +from powerapi.filter import Filter from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ generate_cli_configuration_from_json_file from powerapi.cli.config_parser import SubgroupConfigParser, BaseConfigParser, store_true, RootConfigParser @@ -182,6 +187,66 @@ def several_inputs_outputs_stream_filedb_without_some_arguments_config(several_i return several_inputs_outputs_stream_config +@pytest.fixture +def several_libvirt_pre_processors_config(): + """ + Configuration with several libvirt processors + """ + return load_configuration_from_json_file(file_name='several_libvirt_pre_processors_configuration.json') + + +@pytest.fixture +def several_libvirt_processors_without_some_arguments_config(): + """ + Configuration with several libvirt processors + """ + return load_configuration_from_json_file( + file_name='several_libvirt_processors_without_some_arguments_configuration.json') + + +@pytest.fixture +def several_k8s_pre_processors_config(): + """ + Configuration with several k8s processors + """ + return load_configuration_from_json_file(file_name='several_k8s_pre_processors_configuration.json') + + +@pytest.fixture +def several_k8s_pre_processors_without_some_arguments_config(): + """ + Configuration with several k8s processors + """ + return load_configuration_from_json_file( + file_name='several_k8s_pre_processors_without_some_arguments_configuration.json') + + +@pytest.fixture +def several_k8s_pre_processors(several_k8s_pre_processors_config): + """ + Return a dictionary with several k8s processors + """ + generator = PreProcessorGenerator() + + pre_processors = generator.generate(several_k8s_pre_processors_config) + + return pre_processors + + +@pytest.fixture +def several_k8s_monitors_config(several_k8s_pre_processors): + """ + Configuration with several k8s monitors derived from processor generators + """ + monitors_config = {'monitor': {}} + + for processor_name, processor in several_k8s_pre_processors.items(): + monitors_config['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', + LISTENER_ACTOR_KEY: processor} + + return monitors_config + + @pytest.fixture def csv_io_postmortem_config(invalid_csv_io_stream_config): """ @@ -231,6 +296,51 @@ def config_without_output(csv_io_postmortem_config): return csv_io_postmortem_config +@pytest.fixture +def libvirt_pre_processor_config(): + """ + Configuration with libvirt as pre-processor + """ + return load_configuration_from_json_file(file_name='libvirt_pre_processor_configuration.json') + + +@pytest.fixture +def k8s_pre_processor_config(): + """ + Configuration with k8s as pre-processor + """ + return load_configuration_from_json_file(file_name='k8s_pre_processor_configuration.json') + + +@pytest.fixture +def k8s_pre_processors(k8s_pre_processor_config): + """ + Return a dictionary of k8s pre-processors + """ + generator = PreProcessorGenerator() + + processors = generator.generate(k8s_pre_processor_config) + + return processors + + +@pytest.fixture +def k8s_monitor_config(k8s_pre_processors): + """ + The configuration of k8s monitors is derived from processor generators + """ + + processors = k8s_pre_processors + + monitors = {'monitor': {}} + + for processor_name, processor in processors.items(): + monitors['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', + LISTENER_ACTOR_KEY: processor} + + return monitors + + @pytest.fixture() def subgroup_parser(): """ @@ -444,6 +554,9 @@ def root_config_parsing_manager_with_mandatory_and_optional_arguments(): @pytest.fixture def test_files_path(): + """ + Return the path of directory containing tests files + """ return test_files_module.__path__[0] @@ -452,7 +565,6 @@ def cli_configuration(config_file: str): """ Load in sys.argv a configuration with arguments extracted from a json file """ - # config_file = 'root_manager_basic_configuration.json' sys.argv = generate_cli_configuration_from_json_file(file_name=config_file) yield None @@ -462,8 +574,168 @@ def cli_configuration(config_file: str): @pytest.fixture() def empty_cli_configuration(): + """ + Clean the CLI arguments + """ sys.argv = [] yield None sys.argv = [] + + +@pytest.fixture +def output_input_configuration(): + """ + Return a dictionary containing bindings with a processor + """ + return load_configuration_from_json_file(file_name='output_input_configuration.json') + + +@pytest.fixture(params=['k8s_pre_processor_complete_configuration.json']) +def pre_processor_complete_configuration(request): + """ + Return a dictionary containing a configuration with pre-processor + """ + return load_configuration_from_json_file(file_name=request.param) + + +@pytest.fixture +def pre_processor_config_without_puller(pre_processor_complete_configuration): + """ + Return a configuration with processors but without bindings + """ + + pre_processor_complete_configuration['pre-processor']['my_processor'].pop('puller') + + return pre_processor_complete_configuration + + +@pytest.fixture +def empty_pre_processor_config(pre_processor_complete_configuration): + """ + Return a configuration with bindings but without processors + """ + + pre_processor_complete_configuration.pop('pre-processor') + + return pre_processor_complete_configuration + + +@pytest.fixture(params=['k8s_pre_processor_wrong_binding_configuration.json']) +def pre_processor_wrong_binding_configuration(request): + """ + Return a dictionary containing wrong bindings with a pre-processor + """ + return load_configuration_from_json_file(file_name=request.param) + + +@pytest.fixture(params=['k8s_pre_processor_with_non_existing_puller_configuration.json']) +def pre_processor_with_unexisting_puller_configuration(request): + """ + Return a dictionary containing a pre-processor with a puller that doesn't exist + """ + return load_configuration_from_json_file( + file_name=request.param) + + +@pytest.fixture(params=['k8s_pre_processor_with_reused_puller_in_bindings_configuration.json']) +def pre_processor_with_reused_puller_in_bindings_configuration(request): + """ + Return a dictionary containing a pre-processor with a puller that doesn't exist + """ + return load_configuration_from_json_file( + file_name=request.param) + + +@pytest.fixture +def pre_processor_pullers_and_processors_dictionaries(pre_processor_complete_configuration): + """ + Return a dictionary which contains puller actors, a dictionary of processors as well as the pushers + """ + return get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_complete_configuration) + + +def get_pre_processor_pullers_and_processors_dictionaries_from_configuration(configuration: dict) -> (dict, dict, dict): + """ + Return a tuple of dictionaries (pullers, processors) created from the given configuration. + :param dict configuration : Dictionary containing the configuration + """ + report_filter = Filter() + puller_generator = PullerGenerator(report_filter=report_filter) + pullers = puller_generator.generate(main_config=configuration) + + pusher_generator = PusherGenerator() + pushers = pusher_generator.generate(main_config=configuration) + + route_table = RouteTable() + + dispatcher = DispatcherActor(name='dispatcher', formula_init_function=None, pushers=pushers, + route_table=route_table) + + report_filter.filter(lambda msg: True, dispatcher) + + processor_generator = PreProcessorGenerator() + processors = processor_generator.generate(main_config=configuration) + + return pullers, processors, pushers + + +@pytest.fixture +def dispatcher_actor_in_dictionary(): + """ + Return a DistpatcherActor in a dictionary + """ + + route_table = RouteTable() + + dispatcher = DispatcherActor(name='dispatcher', formula_init_function=None, pushers=None, + route_table=route_table) + + return {dispatcher.name: dispatcher} + + +@pytest.fixture +def pre_processor_binding_manager(pre_processor_pullers_and_processors_dictionaries): + """ + Return a ProcessorBindingManager with a libvirt/K8s Processor + """ + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] + + return PreProcessorBindingManager(pullers=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_binding_configuration): + """ + Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) + """ + _, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_wrong_binding_configuration) + + return PreProcessorBindingManager(pullers=pushers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unexisting_puller_configuration): + """ + Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) + """ + pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_unexisting_puller_configuration) + + return PreProcessorBindingManager(pullers=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_reused_puller_in_bindings( + pre_processor_with_reused_puller_in_bindings_configuration): + """ + Return a PreProcessorBindingManager with a puller used by two different pre-processors + """ + pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_reused_puller_in_bindings_configuration) + + return PreProcessorBindingManager(pullers=pullers, processors=processors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py new file mode 100644 index 00000000..e1bf1b24 --- /dev/null +++ b/tests/unit/cli/test_binding_manager.py @@ -0,0 +1,169 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import copy + +import pytest + +from powerapi.cli.binding_manager import PreProcessorBindingManager +from powerapi.dispatcher import DispatcherActor +from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, \ + TargetActorAlreadyUsed +from powerapi.processor.processor_actor import ProcessorActor + + +def test_create_pre_processor_binding_manager_with_actors(pre_processor_pullers_and_processors_dictionaries): + """ + Test that a PreProcessorBindingManager is correctly created when an actor and a processor dictionary are provided + """ + expected_actors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[0]) + expected_processors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[1]) + + binding_manager = PreProcessorBindingManager(pullers=pre_processor_pullers_and_processors_dictionaries[0], + processors=pre_processor_pullers_and_processors_dictionaries[1]) + + assert binding_manager.actors == expected_actors_dictionary + assert binding_manager.processors == expected_processors_dictionary + + +def test_create_processor_binding_manager_without_actors(): + """ + Test that a ProcessorBindingManager is correctly created without a dictionary + """ + binding_manager = PreProcessorBindingManager(pullers=None, processors=None) + + assert len(binding_manager.actors) == 0 + assert len(binding_manager.processors) == 0 + + +def test_process_bindings_for_pre_processor(pre_processor_complete_configuration, + pre_processor_pullers_and_processors_dictionaries): + """ + Test that the bindings between a puller and a processor are correctly created + """ + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] + + binding_manager = PreProcessorBindingManager(pullers=pullers, + processors=processors) + + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], DispatcherActor) + + binding_manager.process_bindings() + + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], ProcessorActor) + assert pullers['one_puller'].state.report_filter.filters[0][1] == processors['my_processor'] + + +def test_process_bindings_for_pre_processor_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): + """ + Test that an exception is raised with a wrong type for the from actor in a binding + """ + + with pytest.raises(UnsupportedActorTypeException): + pre_processor_binding_manager_with_wrong_binding_types.process_bindings() + + +def test_process_bindings_for_pre_processor_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): + """ + Test that an exception is raised with a puller that doesn't exist + """ + + with pytest.raises(UnexistingActorException): + pre_processor_binding_manager_with_unexisting_puller.process_bindings() + + +def test_process_bindings_for_pre_processor_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): + """ + Test that an exception is raised when the same puller is used by several processors + """ + + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.process_bindings() + + +def test_check_processors_targets_are_unique_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): + """ + Test that an exception is raised when the same puller is used by several processors + """ + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.check_processors_targets_are_unique() + + +def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindings( + pre_processor_binding_manager): + """ + Test that a correct without repeated target passes the validation + """ + try: + pre_processor_binding_manager.check_processors_targets_are_unique() + except TargetActorAlreadyUsed: + assert False + + +def test_check_processor_targets_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): + """ + Test that an exception is raised with a puller that doesn't exist + """ + pre_processor_binding_manager = pre_processor_binding_manager_with_unexisting_puller + with pytest.raises(UnexistingActorException): + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + + +def test_check_processor_targets_raise_exception_with_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): + """ + Test that an exception is raised with a puller that doesn't exist + """ + pre_processor_binding_manager = pre_processor_binding_manager_with_wrong_binding_types + with pytest.raises(UnsupportedActorTypeException): + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + + +def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding_manager): + """ + Test that validation of a configuration with existing targets of the correct type + """ + try: + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + except UnsupportedActorTypeException: + assert False + except UnexistingActorException: + assert False diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py index 08e9af85..2d180312 100644 --- a/tests/unit/cli/test_config_validator.py +++ b/tests/unit/cli/test_config_validator.py @@ -29,7 +29,8 @@ import pytest from powerapi.cli import ConfigValidator -from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException +from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException, \ + UnexistingActorException from tests.utils.cli.base_config_parser import load_configuration_from_json_file @@ -41,16 +42,19 @@ def test_config_in_stream_mode_with_csv_input_raise_an_exception(invalid_csv_io_ ConfigValidator.validate(invalid_csv_io_stream_config) -def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, csv_io_postmortem_config): +def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, + csv_io_postmortem_config): """ Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. The files list for the input has to be transformed into a list """ try: - expected_result = load_configuration_from_json_file(file_name='csv_input_output_stream_mode_enabled_configuration.json') + expected_result = load_configuration_from_json_file( + file_name='csv_input_output_stream_mode_enabled_configuration.json') for current_input in expected_result['input']: if expected_result['input'][current_input]['type'] == 'csv': - expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split(',') + expected_result['input'][current_input]['files'] = ( + expected_result['input'][current_input]['files']).split(',') expected_result['stream'] = False @@ -71,7 +75,8 @@ def test_valid_config_postmortem_csv_input_without_optional_arguments_is_validat expected_result = csv_io_postmortem_config_without_optional_arguments.copy() for current_input in expected_result['input']: if expected_result['input'][current_input]['type'] == 'csv': - expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split(',') + expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split( + ',') expected_result['input'][current_input]['name'] = 'default_puller' expected_result['input'][current_input]['model'] = 'HWPCReport' expected_result['stream'] = False @@ -111,3 +116,57 @@ def test_config_without_outputs_raise_an_exception(config_without_output): ConfigValidator.validate(config_without_output) assert raised_exception.value.argument_name == 'output' + + +def test_config_with_pre_processor_but_without_puller_raise_an_exception(pre_processor_config_without_puller): + """ + Test that validation of a configuration with pre-processors but without a related puller raises a + MissingArgumentException + """ + with pytest.raises(MissingArgumentException) as raised_exception: + ConfigValidator.validate(pre_processor_config_without_puller) + + assert raised_exception.value.argument_name == 'puller' + + +def test_config_with_empty_pre_processor_pass_validation(empty_pre_processor_config): + """ + Test that validation of a configuration without pre-processors passes validation + """ + try: + ConfigValidator.validate(empty_pre_processor_config) + + except MissingArgumentException: + assert False + + +def test_config_with_pre_processor_with_unexisting_puller_actor_raise_an_exception( + pre_processor_with_unexisting_puller_configuration): + """ + Test that validation of a configuration with unexisting actors raise an exception + """ + with pytest.raises(UnexistingActorException) as raised_exception: + ConfigValidator.validate(pre_processor_with_unexisting_puller_configuration) + + assert raised_exception.value.actor == \ + pre_processor_with_unexisting_puller_configuration['pre-processor']['my_processor']['puller'] + + +def test_validation_of_correct_configuration_with_pre_processors(pre_processor_complete_configuration): + """ + Test that a correct configuration with processors and bindings passes the validation + """ + try: + ConfigValidator.validate(pre_processor_complete_configuration) + except Exception: + assert False + + +def test_validation_of_correct_configuration_without_pre_processors_and_bindings(output_input_configuration): + """ + Test that a correct configuration without pre-processors passes the validation + """ + try: + ConfigValidator.validate(output_input_configuration) + except Exception: + assert False diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 3a15b757..9295e7ae 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -1,8 +1,6 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. -import os -import re # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -28,10 +26,19 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +from re import compile, Pattern + import pytest -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator +from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, \ + MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY, PreProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist +from powerapi.processor.pre.k8s.k8s_monitor import K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ + TIMEOUT_QUERY_DEFAULT_VALUE +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor from powerapi.database import MongoDB, CsvDB, SocketDB, InfluxDB, PrometheusDB, InfluxDB2 @@ -46,7 +53,7 @@ def test_generate_puller_from_empty_config_dict_raise_an_exception(): Test that PullerGenerator raises a PowerAPIException when there is no input argument """ conf = {} - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(conf) @@ -56,7 +63,7 @@ def test_generate_puller_from_mongo_basic_config(mongodb_input_output_stream_con """ Test that generation for mongodb puller from a config with a mongodb input works correctly """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(mongodb_input_output_stream_config) @@ -81,7 +88,7 @@ def test_generate_several_pullers_from_config(several_inputs_outputs_stream_conf for _, current_input in several_inputs_outputs_stream_config['input'].items(): if current_input['type'] == 'csv': current_input['files'] = current_input['files'].split(',') - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(several_inputs_outputs_stream_config) assert len(pullers) == len(several_inputs_outputs_stream_config['input']) @@ -110,27 +117,30 @@ def test_generate_several_pullers_from_config(several_inputs_outputs_stream_conf assert False -def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input(several_inputs_outputs_stream_mongo_without_some_arguments_config): +def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input( + several_inputs_outputs_stream_mongo_without_some_arguments_config): """ Test that PullerGenerator raise a PowerAPIException when some arguments are missing for mongo input """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(several_inputs_outputs_stream_mongo_without_some_arguments_config) -def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_actors(several_inputs_outputs_stream_csv_without_some_arguments_config): +def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_actors( + several_inputs_outputs_stream_csv_without_some_arguments_config): """ Test that PullerGenerator generates the csv related actors even if there are some missing arguments """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(several_inputs_outputs_stream_csv_without_some_arguments_config) assert len(pullers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['input']) - for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input'].items(): + for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input']. \ + items(): if current_puller_infos['type'] == 'csv': assert puller_name in pullers @@ -148,7 +158,7 @@ def test_generate_puller_raise_exception_when_missing_arguments_in_socket_input( """ Test that PullerGenerator raise a PowerAPIException when some arguments are missing for socket input """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(several_inputs_outputs_stream_socket_without_some_arguments_config) @@ -171,21 +181,23 @@ def test_remove_model_factory_that_does_not_exist_on_a_DBActorGenerator_must_rai assert len(generator.report_classes) == 5 -def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception(mongodb_input_output_stream_config): +def test_remove_hwpc_report_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception( + mongodb_input_output_stream_config): """ Test that PullGenerator raises PowerAPIException when the model class is not defined """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) generator.remove_report_class('HWPCReport') with pytest.raises(PowerAPIException): _ = generator.generate(mongodb_input_output_stream_config) -def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_input_must_call_sys_exit_(mongodb_input_output_stream_config): +def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_input_must_call_sys_exit_( + mongodb_input_output_stream_config): """ Test that PullGenerator raises a PowerAPIException when an input type is not defined """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) generator.remove_db_factory('mongodb') with pytest.raises(PowerAPIException): _ = generator.generate(mongodb_input_output_stream_config) @@ -279,7 +291,7 @@ def test_generate_several_pushers_from_config(several_inputs_outputs_stream_conf assert db.metric_name == current_pusher_infos['metric_name'] elif pusher_type == 'virtiofs': - assert db.vm_name_regexp == re.compile(current_pusher_infos['vm_name_regexp']) + assert db.vm_name_regexp == compile(current_pusher_infos['vm_name_regexp']) assert db.root_directory_name == current_pusher_infos['root_directory_name'] assert db.vm_directory_name_prefix == current_pusher_infos['vm_directory_name_prefix'] assert db.vm_directory_name_suffix == current_pusher_infos['vm_directory_name_suffix'] @@ -291,7 +303,8 @@ def test_generate_several_pushers_from_config(several_inputs_outputs_stream_conf assert False -def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output(several_inputs_outputs_stream_mongo_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output( + several_inputs_outputs_stream_mongo_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for mongo output """ @@ -301,7 +314,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output( generator.generate(several_inputs_outputs_stream_mongo_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output(several_inputs_outputs_stream_influx_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output( + several_inputs_outputs_stream_influx_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for influx output """ @@ -311,7 +325,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output generator.generate(several_inputs_outputs_stream_influx_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_output(several_inputs_outputs_stream_prometheus_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_output( + several_inputs_outputs_stream_prometheus_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for prometheus output """ @@ -321,7 +336,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_ou generator.generate(several_inputs_outputs_stream_prometheus_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_output(several_inputs_outputs_stream_opentsdb_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_output( + several_inputs_outputs_stream_opentsdb_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for opentsdb output """ @@ -331,7 +347,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_outp generator.generate(several_inputs_outputs_stream_opentsdb_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_output(several_inputs_outputs_stream_virtiofs_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_output( + several_inputs_outputs_stream_virtiofs_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for virtiofs output """ @@ -341,7 +358,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_outp generator.generate(several_inputs_outputs_stream_virtiofs_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output(several_inputs_outputs_stream_filedb_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output( + several_inputs_outputs_stream_filedb_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for filedb output """ @@ -351,7 +369,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output generator.generate(several_inputs_outputs_stream_filedb_without_some_arguments_config) -def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_actors(several_inputs_outputs_stream_csv_without_some_arguments_config): +def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_actors( + several_inputs_outputs_stream_csv_without_some_arguments_config): """ Test that PusherGenerator generates the csv related actors even if there are some missing arguments """ @@ -362,7 +381,8 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a assert len(pushers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['output']) - for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output'].items(): + for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output']. \ + items(): pusher_type = current_pusher_infos['type'] if pusher_type == 'csv': assert pusher_name in pushers @@ -376,3 +396,237 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a generation_checked = True assert generation_checked + + +################################ +# PreProcessorActorGenerator Test # +################################ +def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): + """ + Test that PreProcessGenerator raise an exception when there is no processor argument + """ + conf = {} + generator = PreProcessorGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(conf) + + +@pytest.mark.skip(reason='libvirt is disable by default') +def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config): + """ + Test that generation for libvirt pre-processor from a config works correctly + """ + generator = PreProcessorGenerator() + + processors = generator.generate(libvirt_pre_processor_config) + + assert len(processors) == len(libvirt_pre_processor_config['pre-processor']) + assert 'my_processor' in processors + processor = processors['my_processor'] + + assert isinstance(processor, LibvirtPreProcessorActor) + + assert processor.state.daemon_uri is None + assert isinstance(processor.state.regexp, Pattern) + + +@pytest.mark.skip(reason='libvirt is disable by default') +def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre_processors_config): + """ + Test that several libvirt pre-processors are correctly generated + """ + generator = PreProcessorGenerator() + + processors = generator.generate(several_libvirt_pre_processors_config) + + assert len(processors) == len(several_libvirt_pre_processors_config['pre-processor']) + + for processor_name, current_processor_infos in several_libvirt_pre_processors_config['pre-processor'].items(): + assert processor_name in processors + assert isinstance(processors[processor_name], LibvirtPreProcessorActor) + + assert processors[processor_name].state.daemon_uri is None + assert isinstance(processors[processor_name].state.regexp, Pattern) + + +@pytest.mark.skip(reason='libvirt is disable by default') +def test_generate_libvirt_pre_processor_raise_exception_when_missing_arguments( + several_libvirt_processors_without_some_arguments_config): + """ + Test that PreProcessorGenerator raises a PowerAPIException when some arguments are missing for libvirt processor + """ + generator = PreProcessorGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(several_libvirt_processors_without_some_arguments_config) + + +def check_k8s_pre_processor_infos(pre_processor: K8sPreProcessorActor, expected_pre_processor_info: dict): + """ + Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor + """ + assert isinstance(pre_processor, K8sPreProcessorActor) + + assert pre_processor.state.k8s_api_mode == expected_pre_processor_info["k8s-api-mode"] + assert pre_processor.state.time_interval == expected_pre_processor_info["time-interval"] + assert pre_processor.state.timeout_query == expected_pre_processor_info["timeout-query"] + assert len(pre_processor.state.target_actors) == 0 + assert len(pre_processor.state.target_actors_names) == 1 + assert pre_processor.state.target_actors_names[0] == expected_pre_processor_info["puller"] + + +def test_generate_pre_processor_from_k8s_config(k8s_pre_processor_config): + """ + Test that generation for k8s processor from a config works correctly + """ + generator = PreProcessorGenerator() + processor_name = 'my_processor' + + processors = generator.generate(k8s_pre_processor_config) + + assert len(processors) == len(k8s_pre_processor_config['pre-processor']) + assert processor_name in processors + + processor = processors[processor_name] + + check_k8s_pre_processor_infos(pre_processor=processor, + expected_pre_processor_info=k8s_pre_processor_config["pre-processor"][processor_name]) + + +def test_generate_several_k8s_pre_processors_from_config(several_k8s_pre_processors_config): + """ + Test that several k8s pre-processors are correctly generated + """ + generator = PreProcessorGenerator() + + processors = generator.generate(several_k8s_pre_processors_config) + + assert len(processors) == len(several_k8s_pre_processors_config['pre-processor']) + + for processor_name, current_processor_infos in several_k8s_pre_processors_config['pre-processor'].items(): + assert processor_name in processors + + processor = processors[processor_name] + + check_k8s_pre_processor_infos(pre_processor=processor, expected_pre_processor_info=current_processor_infos) + + +def test_generate_k8s_pre_processor_uses_default_values_with_missing_arguments( + several_k8s_pre_processors_without_some_arguments_config): + """ + Test that PreProcessorGenerator generates a pre-processor with default values when arguments are not defined + """ + generator = PreProcessorGenerator() + + processors = generator.generate(several_k8s_pre_processors_without_some_arguments_config) + + expected_processor_info = {'k8s-api-mode': None, 'time-interval': TIME_INTERVAL_DEFAULT_VALUE, + 'timeout-query': TIMEOUT_QUERY_DEFAULT_VALUE} + + assert len(processors) == len(several_k8s_pre_processors_without_some_arguments_config['pre-processor']) + + pre_processor_number = 1 + for pre_processor_name in several_k8s_pre_processors_without_some_arguments_config['pre-processor']: + assert pre_processor_name in processors + + processor = processors[pre_processor_name] + expected_processor_info['puller'] = 'my_puller_' + str(pre_processor_number) + + check_k8s_pre_processor_infos(pre_processor=processor, expected_pre_processor_info=expected_processor_info) + + pre_processor_number += 1 + + +def check_k8s_monitor_infos(monitor: K8sMonitorAgent, associated_processor: K8sPreProcessorActor): + """ + Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor + """ + + assert isinstance(monitor, K8sMonitorAgent) + + assert monitor.concerned_actor_state.k8s_api_mode == associated_processor.state.k8s_api_mode + + assert monitor.concerned_actor_state.time_interval == associated_processor.state.time_interval + + assert monitor.concerned_actor_state.timeout_query == associated_processor.state.timeout_query + + assert monitor.concerned_actor_state.api_key == associated_processor.state.api_key + + assert monitor.concerned_actor_state.host == associated_processor.state.host + + +def test_generate_k8s_monitor_from_k8s_config(k8s_monitor_config): + """ + Test that generation for k8s monitor from a processor config works correctly + """ + generator = MonitorGenerator() + monitor_name = 'my_processor' + MONITOR_NAME_SUFFIX + + monitors = generator.generate(k8s_monitor_config) + + assert len(monitors) == len(k8s_monitor_config) + + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, + associated_processor=k8s_monitor_config['monitor'][monitor_name][LISTENER_ACTOR_KEY]) + + +def test_generate_several_k8s_monitors_from_config(several_k8s_monitors_config): + """ + Test that several k8s monitors are correctly generated + """ + generator = MonitorGenerator() + + monitors = generator.generate(several_k8s_monitors_config) + + assert len(monitors) == len(several_k8s_monitors_config['monitor']) + + for monitor_name, current_monitor_infos in several_k8s_monitors_config['monitor'].items(): + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, associated_processor=current_monitor_infos[LISTENER_ACTOR_KEY]) + + +def test_generate_k8s_monitor_from_k8s_processors(k8s_pre_processors): + """ + Test that generation for k8s monitor from a processor config works correctly + """ + generator = MonitorGenerator() + processor_name = 'my_processor' + monitor_name = processor_name + MONITOR_NAME_SUFFIX + + monitors = generator.generate_from_processors(processors=k8s_pre_processors) + + assert len(monitors) == len(k8s_pre_processors) + + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, + associated_processor=k8s_pre_processors[processor_name]) + + +def test_generate_several_k8s_monitors_from_processors(several_k8s_pre_processors): + """ + Test that several k8s monitors are correctly generated + """ + generator = MonitorGenerator() + + monitors = generator.generate_from_processors(processors=several_k8s_pre_processors) + + assert len(monitors) == len(several_k8s_pre_processors) + + for processor_name, processor in several_k8s_pre_processors.items(): + monitor_name = processor_name + MONITOR_NAME_SUFFIX + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, associated_processor=processor) diff --git a/tests/unit/processor/__init__.py b/tests/unit/processor/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/conftest.py b/tests/unit/processor/conftest.py new file mode 100644 index 00000000..d973891a --- /dev/null +++ b/tests/unit/processor/conftest.py @@ -0,0 +1,227 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +import multiprocessing +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Any + +from unittest.mock import Mock + +import pytest + +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPodUpdateMetadata, K8sMetadataCacheManager + + +@pytest.fixture(name='pods_list') +def basic_pods_list(): + """ + Return a list of three pods + """ + return ['pod1', 'pod2', 'pod3'] + + +class FakeMetadata: + """ + Fake metadata class related to an event + """ + + def __init__(self, name: str, namespace: str, labels: list): + self.name = name + self.namespace = namespace + self.labels = labels + + +class FakeContainerStatus: + """ + Fake container status infos related to an event + """ + + def __init__(self, container_id: str): + self.container_id = container_id + + +class FakeStatus: + """ + Fake status infos related to an event + """ + + def __init__(self, container_statuses: list): + self.container_statuses = container_statuses + + +class FakePod: + """ + Fake pod class related to an event + """ + + def __init__(self, metadata: FakeMetadata, status: FakeStatus): + self.metadata = metadata + self.status = status + + +@pytest.fixture(name='events_list_k8s') +def basic_events_list_k8s(): + """ + Return a list of three events + """ + return [{ + 'type': 'ADDED', + 'object': FakePod(metadata=FakeMetadata(name='o1', namespace='s1', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[FakeContainerStatus(container_id='test://s1c1'), + FakeContainerStatus(container_id='test://s1c2'), + FakeContainerStatus(container_id='test://s1c3')])) + + }, + { + 'type': 'MODIFIED', + 'object': FakePod(metadata=FakeMetadata(name='o2', namespace='s2', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[FakeContainerStatus(container_id='test://s2c1'), + FakeContainerStatus(container_id='test://s2c2'), + FakeContainerStatus(container_id='test://s2c3')]))}, + { + 'type': 'DELETED', + 'object': FakePod(metadata=FakeMetadata(name='o3', namespace='s3', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[]))} + + ] + + +@pytest.fixture(name='unknown_events_list_k8s') +def basic_unknown_events_list_k8s(events_list_k8s): + """ + Modify and return an event list with unknown events types + """ + event_count = len(events_list_k8s) + + for event_indice in range(event_count): + events_list_k8s[event_indice]['type'] = 'Unknown_' + str(event_indice) + + return events_list_k8s + + +@pytest.fixture(name='expected_events_list_k8s') +def expected_basic_events_list_k8s(events_list_k8s): + """ + Return the expected list of event information according to a list of events + """ + events = [] + + for event_infos in events_list_k8s: + events.append((event_infos['type'], event_infos['object'].metadata.namespace, + event_infos['object'].metadata.name, + extract_containers_ids(event_infos['object'].status.container_statuses), + event_infos['object'].metadata.labels)) + + return events + + +def extract_containers_ids(containers_status: list) -> list: + """ + Return the containers ids by using the given list of containers status + """ + containers_ids = [] + for container_status in containers_status: + containers_ids.append(container_status.container_id[container_status.container_id.find('//') + 2: + len(container_status.container_id)]) + + return containers_ids + + +@pytest.fixture +def expected_k8s_pod_update_metadata(expected_events_list_k8s): + """ + Return a list of K8sPodUpdateMessage by using the provided events list + """ + update_metadata = [] + + for event_type, namespace, name, containers_id, labels in expected_events_list_k8s: + update_metadata.append(K8sPodUpdateMetadata(event=event_type, + namespace=namespace, + pod=name, + containers_id=containers_id, + labels=labels)) + + return update_metadata + + +class MockedWatch(Mock): + """ + Mocked class for simulating the Watch class from K8s API + """ + + def __init__(self, events): + Mock.__init__(self) + self.events = events + self.args = None + self.timeout_seconds = 0 + self.func = None + + def stream(self, func: Any, timeout_seconds: int, *args): + """ + Return the list of events related to the MockedWatch + """ + self.args = args + self.timeout_seconds = timeout_seconds + self.func = func + return self.events + + +@pytest.fixture +def mocked_watch_initialized(events_list_k8s): + """ + Return a MockedWatch with the event list + """ + return MockedWatch(events_list_k8s) + + +@pytest.fixture +def mocked_watch_initialized_unknown_events(unknown_events_list_k8s): + """ + Return a MockedWatch with a list of unknown events + """ + return MockedWatch(unknown_events_list_k8s) + + +class PipeMetadataCacheManager(K8sMetadataCacheManager): + """ + K8sMetadataCacheManager maintains a cache of pods' metadata + (namespace, labels and id of associated containers). + This metadata cache send itself via a pipe when an update is done + """ + + def __init__(self, name: str, level_logger: int, pipe): + K8sMetadataCacheManager.__init__(self, name=name, level_logger=level_logger, + process_manager=multiprocessing.Manager()) + self.pipe = pipe + + def update_cache(self, metadata: K8sPodUpdateMetadata): + """ + Update the local cache for pods. + + Send the metadata cache via the pipe + """ + K8sMetadataCacheManager.update_cache(self, metadata=metadata) + self.pipe.send(self) diff --git a/tests/unit/processor/pre/__init__.py b/tests/unit/processor/pre/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/pre/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/pre/k8s/__init__.py b/tests/unit/processor/pre/k8s/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/pre/k8s/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/pre/k8s/test_k8s_monitor.py b/tests/unit/processor/pre/k8s/test_k8s_monitor.py new file mode 100644 index 00000000..1a251635 --- /dev/null +++ b/tests/unit/processor/pre/k8s/test_k8s_monitor.py @@ -0,0 +1,167 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=R6301,W0613,W0221 + +from time import sleep +from unittest.mock import patch, Mock + +import pytest + +from kubernetes import client + +from powerapi.processor.pre.k8s.k8s_monitor import local_config, MANUAL_CONFIG_MODE, \ + K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPodUpdateMetadata, K8sPreProcessorState, ADDED_EVENT, \ + MODIFIED_EVENT +from powerapi.report import HWPCReport +from tests.utils.actor.dummy_actor import DummyActor + +LISTENER_AGENT_NAME = 'test_k8s_processor_listener_agent' + + +def test_load_local_config(): + """ + Test that load_config works correctly + """ + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock( + return_value={'pod': 'some infos about the pod...'}))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + local_config() + + # Just check we are able to make a request and get a non-empty response + v1_api = client.CoreV1Api() + ret = v1_api.list_pod_for_all_namespaces() + assert ret.items != [] + + +class TestK8sMonitor: + """ + Class for testing a monitor + """ + + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + return K8sPodUpdateMetadata + + @pytest.fixture + def monitor_agent(self, mocked_watch_initialized, pods_list): + """ + Return a monitor agent that uses the provided mocked watch + """ + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): + monitor_agent = K8sMonitorAgent(name='test_k8s_monitor', + concerned_actor_state=K8sPreProcessorState( + actor=DummyActor(name='test_k8s_monitor_actor', + pipe=None, message_type=HWPCReport), + target_actors=[], + target_actors_names=[], + k8s_api_mode=MANUAL_CONFIG_MODE, + time_interval=10, + timeout_query=10, + api_key='', + host='' + ) + ) + yield monitor_agent + + def test_streaming_query(self, monitor_agent, pods_list, expected_events_list_k8s, mocked_watch_initialized, + shutdown_system): + """ + Test that k8s_streaming_query is able to retrieve events related to pods + """ + result = monitor_agent.k8s_streaming_query() + + assert result == expected_events_list_k8s + + def test_unknown_events_streaming_query(self, pods_list, mocked_watch_initialized_unknown_events, + monitor_agent, shutdown_system): + """ + Test that unknown events are ignored by k8s_streaming_query + """ + result = monitor_agent.k8s_streaming_query() + + assert result == [] + + def test_monitor_agent_update_metadata_cache_when_events_are_available(self, monitor_agent, + expected_k8s_pod_update_metadata, + shutdown_system): + """ + Test that the monitor updates metadata cache when events are available + """ + expected_pods_labels_size = 0 + expected_containers_ids_size = 0 + for current_metadata in expected_k8s_pod_update_metadata: + if current_metadata.event in [ADDED_EVENT, MODIFIED_EVENT]: + expected_pods_labels_size += 1 + expected_containers_ids_size += len(current_metadata.containers_id) + + monitor_agent.start() + + sleep(1) + + monitor_agent.stop_monitoring.set() + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) == \ + expected_pods_labels_size # There is a event if each type ADDED, DELETED and MODIFIED + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) == \ + expected_pods_labels_size + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) == \ + expected_containers_ids_size + + assert monitor_agent.stop_monitoring.is_set() + + def test_stop_monitor_agent_works(self, monitor_agent): + """ + Test that monitor agent is correctly stopped when the flag related to the monitoring is changed + """ + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) == 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) == 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) == 0 + assert not monitor_agent.stop_monitoring.is_set() + + monitor_agent.start() + + sleep(1) + + monitor_agent.stop_monitoring.set() + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) > 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) > 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) > 0 + assert monitor_agent.stop_monitoring.is_set() diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py new file mode 100644 index 00000000..e7be41a4 --- /dev/null +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -0,0 +1,384 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=R6301,W0221,W0613 + +import logging +from copy import deepcopy +from time import sleep +from unittest.mock import patch, Mock + +import pytest + +from powerapi.processor.pre.k8s.k8s_monitor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT, \ + K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, K8sPodUpdateMetadata +from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import clean_up_container_id, POD_NAMESPACE_METADATA_KEY, \ + POD_NAME_METADATA_KEY +from powerapi.report import HWPCReport +from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe +from tests.unit.processor.conftest import MockedWatch, FakePod, FakeMetadata, FakeStatus, FakeContainerStatus +from tests.utils.report.hwpc import extract_rapl_reports_with_2_sockets + +DISPATCHER_NAME = 'test_k8s_processor_dispatcher' + +KUBERNETES_CLIENT_API_REFERENCE = 'kubernetes.client.CoreV1Api' + +KUBERNETES_LOAD_CONFIG_REFERENCE = 'kubernetes.config.load_kube_config' + +KUBERNETES_WATCH_REFERENCE = 'kubernetes.watch.Watch' + + +def get_metadata_from_event(basic_event: dict): + """ + Create a K8sPodUpdateMetadata from a dict containing the event info + :param dict basic_event : The event information + """ + containers_id = [] + + for current_fake_container_status in basic_event['object'].status.container_statuses: + index_id = current_fake_container_status.container_id.index('://') + current_container_id = current_fake_container_status.container_id[index_id + + len('://'): + len(current_fake_container_status.container_id + )] + containers_id.append(current_container_id) + + return K8sPodUpdateMetadata(event=basic_event['type'], + namespace=basic_event['object'].metadata.namespace, + pod=basic_event['object'].metadata.name, + containers_id=containers_id, + labels=basic_event['object'].metadata.labels) + + +def test_clean_up_id_docker(): + """ + Test that the cleanup of the docker id works correctly + """ + r = clean_up_container_id( + "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod435532e3_546d_45e2_8862_d3c7b320d2d9.slice/" + "docker-68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a.scope") + + assert r == "68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a" + + +def test_clean_up_id_othercri(): + """ + Test that the cleanup of the docker id works correctly + """ + r = clean_up_container_id( + "/kubepods/besteffort/pod42006d2c-cad7-4575-bfa3-91848a558743/ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5") + + assert r == "ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5" + + +class TestK8sProcessor(AbstractTestActor): + """ + Class for testing a K8s Processor Actor + """ + + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + return HWPCReport + + @pytest.fixture + def basic_added_event_k8s(self): + """ + Return a basic ADDED event and its related information + """ + return { + 'type': ADDED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels={'l1': 'v1', 'l2': 'v2', 'l3': 'v3', 'l4': 'v4', 'l5': 'v5'}), + status=FakeStatus(container_statuses=[FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_1_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_2_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_3_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_4_added')]) + ) + } + + @pytest.fixture + def basic_modified_event_k8s(self): + """ + Return a basic MODIFIED event and its related information + """ + return { + 'type': MODIFIED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels={'l1_m': 'v1', 'l2_m': 'v2', 'l3_m': 'v3', 'l4_m': 'v4', 'l5_m': 'v5'}), + status=FakeStatus(container_statuses=[FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_1_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_2_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_3_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_4_modified')]) + ) + } + + @pytest.fixture + def basic_deleted_event_k8s(self): + """ + Return a basic DELETED event and its related information + """ + return { + 'type': DELETED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels=[]), + status=None + ) + } + + @pytest.fixture + def basic_unknown_event_k8s(self): + """ + Return a basic DELETED event and its related information + """ + return { + 'type': 'Unknown Event', + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels=[]), + status=None + ) + } + + @pytest.fixture() + def hwpc_report(self, basic_added_event_k8s): + """ + Return a HWPC Report + """ + json_input = extract_rapl_reports_with_2_sockets(1)[0] + report = HWPCReport.from_json(json_input) + report.target = basic_added_event_k8s['object'].status.container_statuses[0].container_id + + return report + + @pytest.fixture() + def hwpc_report_with_metadata(self, hwpc_report, basic_added_event_k8s): + """ + Return a HWPC report with metadata + """ + update_metadata_cache_added_event = get_metadata_from_event(basic_event=basic_added_event_k8s) + hwpc_report_with_metadata = deepcopy(hwpc_report) + + hwpc_report_with_metadata.metadata[POD_NAMESPACE_METADATA_KEY] = \ + update_metadata_cache_added_event.namespace + hwpc_report_with_metadata.metadata[POD_NAME_METADATA_KEY] = update_metadata_cache_added_event.pod + + for label_name, label_value in update_metadata_cache_added_event.labels.items(): + hwpc_report_with_metadata.metadata[f"label_{label_name}"] = label_value + + return hwpc_report_with_metadata + + @pytest.fixture + def actor(self, started_fake_target_actor, pods_list, mocked_watch_initialized): + return K8sPreProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, + target_actors=[started_fake_target_actor], + level_logger=logging.DEBUG) + + @pytest.fixture + def mocked_monitor_added_event(self, actor, basic_added_event_k8s, pods_list): + """ + Return a mocked monitor that produces an added event + """ + with patch(KUBERNETES_CLIENT_API_REFERENCE, + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, + return_value=MockedWatch(events=[basic_added_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + @pytest.fixture + def mocked_monitor_modified_event(self, actor, basic_modified_event_k8s, pods_list): + """ + Return a mocked monitor that produces a modified event + """ + with patch(KUBERNETES_CLIENT_API_REFERENCE, + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, + return_value=MockedWatch(events=[basic_modified_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + @pytest.fixture + def mocked_monitor_deleted_event(self, actor, basic_deleted_event_k8s, pods_list): + """ + Return a mocked monitor that produces a deleted event + """ + with patch(KUBERNETES_CLIENT_API_REFERENCE, + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, + return_value=MockedWatch(events=[basic_deleted_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + @pytest.fixture + def mocked_monitor_unknown_event(self, actor, basic_added_event_k8s, basic_unknown_event_k8s, pods_list): + """ + Return a mocked monitor that produces an unknown event + """ + with patch(KUBERNETES_CLIENT_API_REFERENCE, + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, + return_value=MockedWatch(events=[basic_unknown_event_k8s, basic_added_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + def test_update_metadata_cache_with_added_event(self, mocked_monitor_added_event, started_actor, + basic_added_event_k8s, shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of an added event + """ + metadata_for_update = get_metadata_from_event(basic_event=basic_added_event_k8s) + mocked_monitor_added_event.start() + sleep(1) + + result = started_actor.state.metadata_cache_manager + + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id + + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) + + mocked_monitor_added_event.stop_monitoring.set() + + def test_update_metadata_cache_with_modified_event(self, mocked_monitor_modified_event, started_actor, + basic_modified_event_k8s, shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of a modified event + """ + + metadata_for_update = get_metadata_from_event(basic_event=basic_modified_event_k8s) + mocked_monitor_modified_event.start() + sleep(10) + + result = started_actor.state.metadata_cache_manager + + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id + + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) + + mocked_monitor_modified_event.stop_monitoring.set() + + def test_update_metadata_cache_with_deleted_event(self, mocked_monitor_deleted_event, started_actor, + basic_deleted_event_k8s, shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of a deleted event + """ + mocked_monitor_deleted_event.start() + sleep(0.5) + + result = started_actor.state.metadata_cache_manager + + assert len(result.pod_labels) == 0 + assert len(result.pod_containers) == 0 + assert len(result.containers_pod) == 0 + + mocked_monitor_deleted_event.stop_monitoring.set() + + def test_update_metadata_cache_with_unknown_event_does_not_modify_it(self, mocked_monitor_unknown_event, + started_actor, + basic_added_event_k8s, + shutdown_system): + """ + Test that metadata_cache is not updated when a reception of an unknown event + """ + metadata_for_update = get_metadata_from_event(basic_event=basic_added_event_k8s) + mocked_monitor_unknown_event.start() + + sleep(1) + + result = started_actor.state.metadata_cache_manager + + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id + + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) + + mocked_monitor_unknown_event.stop_monitoring.set() + + def test_add_metadata_to_hwpc_report(self, mocked_monitor_added_event, + started_actor, + hwpc_report, hwpc_report_with_metadata, + dummy_pipe_out, + shutdown_system): + """ + Test that a HWPC report is modified with the correct metadata + """ + mocked_monitor_added_event.start() + + sleep(1) + + started_actor.send_data(hwpc_report) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result[1] == hwpc_report_with_metadata + + mocked_monitor_added_event.stop_monitoring.set() + + def test_add_metadata_to_hwpc_report_does_not_modify_report_with_unknown_container_id(self, + mocked_monitor_added_event, + started_actor, + hwpc_report, + dummy_pipe_out, + shutdown_system): + """ + Test that a HWPC report is not modified with an unknown container id + """ + + started_actor.send_data(hwpc_report) + + result = recv_from_pipe(dummy_pipe_out, 4) + + assert result[1] == hwpc_report diff --git a/tests/unit/processor/pre/libvirt/__init__.py b/tests/unit/processor/pre/libvirt/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/pre/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/pre/libvirt/test_libvirt_processor.py b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py new file mode 100644 index 00000000..2fa21801 --- /dev/null +++ b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py @@ -0,0 +1,103 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=no-self-use,arguments-differ,unused-argument + +from time import sleep +import pytest +from mock.mock import patch + +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor +from powerapi.report import Report +from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe +from tests.utils.actor.dummy_actor import DummyActor +from tests.utils.libvirt import REGEXP, LIBVIRT_TARGET_NAME1, UUID_1, MockedLibvirt, LIBVIRT_TARGET_NAME2 + + +BAD_TARGET = 'lkjqlskjdlqksjdlkj' +DISPATCHER_NAME = 'test_libvirt_processor_dispatcher' +REPORT_TYPE_TO_BE_SENT = Report + + +class TestLibvirtProcessor(AbstractTestActor): + """ + Class to test the processor related to libvirt + """ + @pytest.fixture + def started_fake_dispatcher(self, dummy_pipe_in): + """ + Return a started DummyActor. When the test is finished, the actor is stopped + """ + dispatcher = DummyActor(name=DISPATCHER_NAME, pipe=dummy_pipe_in, message_type=REPORT_TYPE_TO_BE_SENT) + dispatcher.start() + + yield dispatcher + if dispatcher.is_alive(): + dispatcher.terminate() + + @pytest.fixture + def actor(self, started_fake_dispatcher): + with patch('powerapi.processor.pre.libvirt.libvirt_pre_processor_actor.openReadOnly', + return_value=MockedLibvirt()): + return LibvirtPreProcessorActor(name='processor_actor', uri='', regexp=REGEXP, + target_actors=[started_fake_dispatcher]) + + @pytest.mark.skip(reason='libvirt is disable by default') + def test_modify_report_that_not_match_regexp_must_not_modify_report(self, started_actor, + dummy_pipe_out, + shutdown_system): + """ + Test that te LibvirtProcessorActor does not modify an report that does not match the regexp + """ + report = Report(0, 'sensor', BAD_TARGET) + started_actor.send_data(msg=report) + sleep(1) + assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) + + @pytest.mark.skip(reason='libvirt is disable by default') + def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): + """ + Test that a report matching the regexp of the processor is actually modified + """ + report = Report(0, 'sensor', LIBVIRT_TARGET_NAME1) + started_actor.send_data(msg=report) + new_report = recv_from_pipe(dummy_pipe_out, 2)[1] + assert new_report.metadata["domain_id"] == UUID_1 + + @pytest.mark.skip(reason='libvirt is disable by default') + def test_modify_report_that_match_regexp_but_with_wrong_domain_name_must_not_modify_report(self, started_actor, + dummy_pipe_out, + shutdown_system): + """ + Test that a report matching the regexp but with wrong domain name is not modified by the processor + """ + report = Report(0, 'sensor', LIBVIRT_TARGET_NAME2) + started_actor.send_data(msg=report) + sleep(1) + assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) diff --git a/tests/unit/puller/simple/test_simple_puller_actor.py b/tests/unit/puller/simple/test_simple_puller_actor.py index 0ddd2f8f..e9eb2f48 100644 --- a/tests/unit/puller/simple/test_simple_puller_actor.py +++ b/tests/unit/puller/simple/test_simple_puller_actor.py @@ -64,7 +64,7 @@ class TestSimplePuller(AbstractTestActor): @pytest.fixture def started_fake_dispatcher(self, dummy_pipe_in): """ - Return a started DummyActor. When the test is finished, the actor is stopped + Return a started DummyActor. When the test is finished, the actor is stopped """ dispatcher = DummyActor(DISPATCHER_NAME, dummy_pipe_in, REPORT_TYPE_TO_BE_SENT) dispatcher.start() @@ -75,7 +75,7 @@ def started_fake_dispatcher(self, dummy_pipe_in): @pytest.fixture def fake_filter(self, started_fake_dispatcher): """ - Return a fake filter for a started dispatcher. The use rule always returns True + Return a fake filter for a started dispatcher. The use rule always returns True """ fake_filter = Filter() fake_filter.filter(filter_rule, started_fake_dispatcher) @@ -84,7 +84,7 @@ def fake_filter(self, started_fake_dispatcher): @pytest.fixture def empty_filter(self): """ - Return a filter withour rules + Return a filter withour rules """ fake_filter = Filter() return fake_filter @@ -97,7 +97,7 @@ def actor(self, fake_filter): @pytest.fixture def actor_without_rules(self, empty_filter): """ - Return a SimplePullerActor with a empty filter + Return a SimplePullerActor with a empty filter """ return SimplePullerActor(name=ACTOR_NAME, number_of_reports_to_send=NUMBER_OF_REPORTS_TO_SEND, report_type_to_send=REPORT_TYPE_TO_BE_SENT, report_filter=empty_filter) @@ -105,8 +105,8 @@ def actor_without_rules(self, empty_filter): @pytest.fixture def init_actor_without_rules(self, actor_without_rules): """ - Return an initialized actor, i.e., started and with data and control sockets connected. At the end of the - test, the actor is stopped + Return an initialized actor, i.e., started and with data and control sockets connected. At the end of the + test, the actor is stopped """ actor_without_rules.start() actor_without_rules.connect_data() @@ -119,7 +119,7 @@ def init_actor_without_rules(self, actor_without_rules): @pytest.fixture def init_actor_without_terminate(self, actor): """ - Return an initialized actor, i.e., started and with data and control sockets connected + Return an initialized actor, i.e., started and with data and control sockets connected """ actor.start() actor.connect_data() @@ -128,7 +128,7 @@ def init_actor_without_terminate(self, actor): def test_create_simple_puller_without_rules_is_no_initialized(self, init_actor_without_rules, shutdown_system): """ - Check that a SimplePuller without rules is no initialized + Check that a SimplePuller without rules is no initialized """ init_actor_without_rules.send_control(StartMessage('system')) @@ -141,7 +141,7 @@ def test_start_actor_send_reports_to_dispatcher(self, shutdown_system): """ - Check that a SimplePuller sends reports to dispatcher + Check that a SimplePuller sends reports to dispatcher """ count = 0 report = REPORT_TYPE_TO_BE_SENT.create_empty_report() @@ -158,7 +158,7 @@ def test_starting_actor_terminate_itself_after_poison_message_reception(self, in shutdown_system): """ - Check that a SimplePuller stops when it receives a PoisonPillMessage + Check that a SimplePuller stops when it receives a PoisonPillMessage """ init_actor_without_terminate.send_control(PoisonPillMessage('simple-test-simple-puller')) assert not is_actor_alive(init_actor_without_terminate) diff --git a/tests/utils/acceptation.py b/tests/utils/acceptation.py index aa35052b..5155fef6 100644 --- a/tests/utils/acceptation.py +++ b/tests/utils/acceptation.py @@ -36,7 +36,7 @@ import pytest from powerapi.actor import Supervisor -from powerapi.cli.generator import PusherGenerator, PullerGenerator, ReportModifierGenerator +from powerapi.cli.generator import PusherGenerator, PullerGenerator from powerapi.dispatch_rule import HWPCDispatchRule, HWPCDepthLevel from powerapi.dispatcher import RouteTable, DispatcherActor from powerapi.filter import Filter @@ -134,7 +134,7 @@ def get_basic_config_with_stream(): def launch_simple_architecture(config: Dict, supervisor: Supervisor, hwpc_depth_level: str, - formula_class: Callable, generate_report_modifier_list=False): + formula_class: Callable): """ Launch a simple architecture with a pusher, a dispatcher et a puller. :param config: Architecture configuration @@ -169,12 +169,7 @@ def launch_simple_architecture(config: Dict, supervisor: Supervisor, hwpc_depth_ report_filter = Filter() report_filter.filter(filter_rule, dispatcher) - report_modifier_list = [] - if generate_report_modifier_list: - report_modifier_generator = ReportModifierGenerator() - report_modifier_list = report_modifier_generator.generate(config) - - puller_generator = PullerGenerator(report_filter, report_modifier_list) + puller_generator = PullerGenerator(report_filter) puller_info = puller_generator.generate(config) puller = puller_info['test_puller'] supervisor.launch_actor(actor=puller, start_message=True) diff --git a/tests/utils/actor/dummy_handlers.py b/tests/utils/actor/dummy_handlers.py index 8f03040c..de9b773c 100644 --- a/tests/utils/actor/dummy_handlers.py +++ b/tests/utils/actor/dummy_handlers.py @@ -48,4 +48,4 @@ def handle(self, msg: Message): :param Object msg: the message received by the actor """ self.state.pipe.send((self.state.actor.name, msg)) - logging.debug('receive : ' + str(msg), extra={'actor_name': self.state.actor.name}) + logging.debug('received : ' + str(msg), extra={'actor_name': self.state.actor.name}) diff --git a/tests/utils/cli/k8s_pre_processor_complete_configuration.json b/tests/utils/cli/k8s_pre_processor_complete_configuration.json new file mode 100644 index 00000000..b637d766 --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_complete_configuration.json @@ -0,0 +1,31 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 60, + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/k8s_pre_processor_configuration.json b/tests/utils/cli/k8s_pre_processor_configuration.json new file mode 100644 index 00000000..3d50a133 --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_configuration.json @@ -0,0 +1,12 @@ +{ + "verbose": true, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 20, + "timeout-query": 30, + "puller": "my_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json new file mode 100644 index 00000000..5f979837 --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json @@ -0,0 +1,31 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, + "puller": "non_existent_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json new file mode 100644 index 00000000..b447bd92 --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json @@ -0,0 +1,38 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, + "puller": "one_puller" + }, + "my_processor_2": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json new file mode 100644 index 00000000..c3198210 --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json @@ -0,0 +1,31 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 70, + "timeout-query": 80, + "puller": "one_pusher" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_complete_configuration.json b/tests/utils/cli/libvirt_pre_processor_complete_configuration.json new file mode 100644 index 00000000..a9a0596b --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_complete_configuration.json @@ -0,0 +1,30 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_configuration.json b/tests/utils/cli/libvirt_pre_processor_configuration.json new file mode 100644 index 00000000..362fbba9 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_configuration.json @@ -0,0 +1,11 @@ +{ + "verbose": true, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "my_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json b/tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json new file mode 100644 index 00000000..0f7f9466 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json @@ -0,0 +1,30 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "non_existent_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json new file mode 100644 index 00000000..42e60e29 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json @@ -0,0 +1,36 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_puller" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json b/tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json new file mode 100644 index 00000000..9b3a8558 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json @@ -0,0 +1,30 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_pusher" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/output_input_configuration.json b/tests/utils/cli/output_input_configuration.json new file mode 100644 index 00000000..5c681b90 --- /dev/null +++ b/tests/utils/cli/output_input_configuration.json @@ -0,0 +1,22 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_pre_processors_configuration.json b/tests/utils/cli/several_k8s_pre_processors_configuration.json new file mode 100644 index 00000000..48112c4e --- /dev/null +++ b/tests/utils/cli/several_k8s_pre_processors_configuration.json @@ -0,0 +1,47 @@ +{ + "verbose": true, + "pre-processor": { + "my_processor_1": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 20, + "timeout-query": 20, + "puller": "my_puller_1" + }, + "my_processor_2": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 30, + "timeout-query": 30, + "puller": "my_puller_2" + }, + "my_processor_3": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 40, + "timeout-query": 40, + "puller": "my_puller_3" + }, + "my_processor_4": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 50, + "puller": "my_puller_4" + }, + "my_processor_5": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 60, + "timeout-query": 60, + "puller": "my_puller_5" + }, + "my_processor_6": { + "type": "k8s", + "k8s-api-mode": "Manual", + "time-interval": 70, + "timeout-query": 70, + "puller": "my_puller_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json new file mode 100644 index 00000000..1f068c53 --- /dev/null +++ b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json @@ -0,0 +1,29 @@ +{ + "verbose": true, + "pre-processor": { + "my_processor_1": { + "type": "k8s", + "puller": "my_puller_1" + }, + "my_processor_2": { + "type": "k8s", + "puller": "my_puller_2" + }, + "my_processor_3": { + "type": "k8s", + "puller": "my_puller_3" + }, + "my_processor_4": { + "type": "k8s", + "puller": "my_puller_4" + }, + "my_processor_5": { + "type": "k8s", + "puller": "my_puller_5" + }, + "my_processor_6": { + "type": "k8s", + "puller": "my_puller_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_libvirt_pre_processors_configuration.json b/tests/utils/cli/several_libvirt_pre_processors_configuration.json new file mode 100644 index 00000000..52bf2a3d --- /dev/null +++ b/tests/utils/cli/several_libvirt_pre_processors_configuration.json @@ -0,0 +1,41 @@ +{ + "verbose": true, + "pre-processor": { + "my_processor_1": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_1", + "puller": "my_puller_1" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_2", + "puller": "my_puller_2" + }, + "my_processor_3": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_3", + "puller": "my_puller_3" + }, + "my_processor_4": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_4", + "puller": "my_puller_4" + }, + "my_processor_5": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_5", + "puller": "my_puller_5" + }, + "my_processor_6": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_6", + "puller": "my_puller_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json new file mode 100644 index 00000000..a86bdec6 --- /dev/null +++ b/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json @@ -0,0 +1,32 @@ +{ + "processor": { + "my_processor_1": { + "type": "libvirt", + "regexp": "a_reg_exp_1" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "" + }, + "my_processor_3": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_3" + }, + "my_processor_4": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_4" + }, + "my_processor_5": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_5" + }, + "my_processor_6": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/libvirt.py b/tests/utils/libvirt.py index 8e0bddf3..4963c6d6 100644 --- a/tests/utils/libvirt.py +++ b/tests/utils/libvirt.py @@ -32,7 +32,11 @@ try: from libvirt import libvirtError except ImportError: - from powerapi.report_modifier.libvirt_mapper import LibvirtException + class LibvirtException(Exception): + """""" + + def __init__(self, _): + Exception.__init__(self) libvirtError = LibvirtException DOMAIN_NAME_1 = 'instance-00000001'