From c83cb4bad5244bc3278e57e453853b0279766372 Mon Sep 17 00:00:00 2001 From: daniel Date: Wed, 26 Jul 2023 16:28:47 +0200 Subject: [PATCH 01/35] feat: Add ProcessorActor an LibvirtProcessorActor based on LibvirtMapper. Add binding manager for this kind of actor --- powerapi/cli/binding_manager.py | 221 ++++++++++++++++++ powerapi/cli/generator.py | 62 ++++- powerapi/dispatcher/dispatcher_actor.py | 15 ++ powerapi/exception.py | 40 ++++ powerapi/processor/__init__.py | 28 +++ powerapi/processor/handlers.py | 62 +++++ powerapi/processor/k8s/__init__.py | 28 +++ powerapi/processor/libvirt/__init__.py | 28 +++ .../libvirt/libvirt_processor_actor.py | 80 +++++++ .../libvirt/libvirt_processor_handlers.py | 82 +++++++ powerapi/processor/processor_actor.py | 86 +++++++ powerapi/puller/simple/simple_puller_actor.py | 12 +- tests/unit/cli/conftest.py | 125 ++++++++++ tests/unit/cli/test_binding_manager.py | 150 ++++++++++++ tests/unit/cli/test_generator.py | 115 +++++++-- tests/unit/processor/__init__.py | 28 +++ tests/unit/processor/libvirt/__init__.py | 28 +++ .../libvirt/test_libvirt_processor.py | 99 ++++++++ ...ith_non_existent_puller_configuration.json | 35 +++ .../cli/libvirt_processor_configuration.json | 9 + ...bvirt_processor_binding_configuration.json | 35 +++ ...processor_wrong_binding_configuration.json | 35 +++ ...eral_libvirt_processors_configuration.json | 34 +++ ..._without_some_arguments_configuration.json | 32 +++ 24 files changed, 1441 insertions(+), 28 deletions(-) create mode 100644 powerapi/cli/binding_manager.py create mode 100644 powerapi/processor/__init__.py create mode 100644 powerapi/processor/handlers.py create mode 100644 powerapi/processor/k8s/__init__.py create mode 100644 powerapi/processor/libvirt/__init__.py create mode 100644 powerapi/processor/libvirt/libvirt_processor_actor.py create mode 100644 powerapi/processor/libvirt/libvirt_processor_handlers.py create mode 100644 powerapi/processor/processor_actor.py create mode 100644 tests/unit/cli/test_binding_manager.py create mode 100644 tests/unit/processor/__init__.py create mode 100644 tests/unit/processor/libvirt/__init__.py create mode 100644 tests/unit/processor/libvirt/test_libvirt_processor.py create mode 100644 tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json create mode 100644 tests/utils/cli/libvirt_processor_configuration.json create mode 100644 tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json create mode 100644 tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json create mode 100644 tests/utils/cli/several_libvirt_processors_configuration.json create mode 100644 tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py new file mode 100644 index 00000000..80dbb827 --- /dev/null +++ b/powerapi/cli/binding_manager.py @@ -0,0 +1,221 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from powerapi.actor import Actor +from powerapi.exception import MissingArgumentException, MissingValueException, BadInputData, \ + UnsupportedActorTypeException +from powerapi.processor.processor_actor import ProcessorActor +from powerapi.puller import PullerActor +from powerapi.pusher import PusherActor + +FROM_KEY = 'from' +TO_KEY = 'to' +PATH_SEPARATOR = '.' +INPUT_GROUP = 'input' +OUTPUT_GROUP = 'output' +PROCESSOR_GROUP = 'processor' +BINDING_GROUP = 'binding' + + +class BindingManager: + """ + Class for management the binding between actors during their creation process + """ + + def __init__(self, actors: dict = {}): + """ + :param dict actors: Dictionary of actors to create the bindings + """ + self.actors = actors + + def process_bindings(self, bindings: dict): + """ + Define bindings between self.actors according to the provided binding dictionary. + """ + raise NotImplementedError() + + +def check_parameters_in_binding(binding: dict): + """ + Check that a binding has the from and to parameters and that they have a value. + If it is not the case, it raises a MissingArgumentException or + """ + # Check from and to exist + if FROM_KEY not in binding: + raise MissingArgumentException(argument_name=FROM_KEY) + + if TO_KEY not in binding: + raise MissingArgumentException(argument_name=TO_KEY) + + from_actor_path = binding[FROM_KEY] + to_actor_path = binding[TO_KEY] + + # Check that from and to values are not empty + if from_actor_path == "" or to_actor_path == "": + raise MissingValueException(argument_name=FROM_KEY + ' or ' + TO_KEY) + + +class ProcessorBindingManager(BindingManager): + """ + Class for management the binding between processor actors and others actors + """ + + def __init__(self, actors: dict): + """ + The ProcessorBindingManager keeps an actor dictionary with the following structure: [][actor_name] + where is 'input' for pullers, 'output' for pushers and + 'processor' for processors + :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} + """ + + BindingManager.__init__(self, actors={INPUT_GROUP: {}, OUTPUT_GROUP: {}, PROCESSOR_GROUP: {}}) + if actors: + self.add_actors(actors=actors) + + def process_bindings(self, bindings: dict): + """ + Define bindings between self.actors according to the provided binding dictionary. + This dictionary has the structure + {"": + "from": "", + "to": "" + } + the "" and "to": "" follow the convention "." + according to the configuration, e.g., "input.my_puller" and "processor.my_libvirt_processor" + + One of the actors in the binding hs to be a processor. If the "to" actor is the processor, the "from" has to be + a puller. If the "from" actor is a processor, the "to" actor has to be a pusher. + :param bindings: The bindings to be processed. + """ + + for _, binding in bindings.items(): + + check_parameters_in_binding(binding=binding) + + from_actor_path = binding[FROM_KEY] + to_actor_path = binding[TO_KEY] + + # Check that the paths have the correct format and the actors with the + # given paths exist + + from_actor_path = self.check_actor_path(actor_path=from_actor_path) + to_actor_path = self.check_actor_path(actor_path=to_actor_path) + + # Check that actors types are correct + + from_actor = self.actors[from_actor_path[0]][from_actor_path[1]] + to_actor = self.actors[to_actor_path[0]][to_actor_path[1]] + + # Check types and do the processing + if isinstance(from_actor, ProcessorActor): + if not isinstance(to_actor, PusherActor): + raise BadInputData() + + # The processor has to be between the formula and the pusher + # The pusher becomes a target of the processor + processor = from_actor + processor.add_target_actor(actor=to_actor) + + # We look for the pusher on each dispatcher in order to replace it by + # the processor. + for _, puller in self.actors[INPUT_GROUP]: + for filter in puller.state.report_filter.filters: + dispatcher = filter[1] + + number_of_pushers = len(dispatcher.pusher) + pusher_updated = False + + for index in range(number_of_pushers): + if dispatcher.pusher[index] == to_actor: + dispatcher.pusher[index] = processor + pusher_updated = True + break + + if pusher_updated: + dispatcher.update_state_formula_factory() + + elif isinstance(to_actor, ProcessorActor): + if not isinstance(from_actor, PullerActor): + raise BadInputData() + + # The processor has to be between the puller and the dispatcher + # The dispatcher becomes a target of the processor + + # The dispatcher defines the relationship between the Formula and + # puller + processor = to_actor + number_of_filters = len(from_actor.state.report_filter.filters) + + for index in range(number_of_filters): + # The filters define the relationship with the dispatcher + # The relationship has to be updated + current_filter = list(from_actor.state.report_filter.filters[index]) + current_filter_dispatcher = current_filter[1] + processor.add_target_actor(actor=current_filter_dispatcher) + current_filter[1] = processor + from_actor.state.report_filter.filters[index] = tuple(current_filter) + else: + raise BadInputData() + + def add_actor(self, actor: Actor): + """ + Add the actor to the dictionary of actors according to its type. + Actor has to be PullerActor, PusherActor or ProcessorActor. The key of the actor is its name + """ + group = None + if isinstance(actor, PullerActor): + group = INPUT_GROUP + elif isinstance(actor, PusherActor): + group = OUTPUT_GROUP + elif isinstance(actor, ProcessorActor): + group = PROCESSOR_GROUP + else: + raise UnsupportedActorTypeException(actor_type=str(type(actor))) + + self.actors[group][actor.name] = actor + + def add_actors(self, actors: dict): + """ + Add the dictionary of actors to the manager dictionary + """ + for _, actor in actors.items(): + self.add_actor(actor) + + def check_actor_path(self, actor_path: str): + """ + Check that an actor path is separated by PATH_SEPARATOR, that it has to subpaths (group and actor name) + and the actor exist in self.actors. It raises a BadInputData exception is these conditions are not respected. + Otherwise, it returns the path in a list with two elements + """ + + path = actor_path.split(PATH_SEPARATOR) + + if len(path) != 2 or path[0] not in self.actors or path[1] not in self.actors[path[0]]: + raise BadInputData() + + return path diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index 3102fe45..d3903b8d 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -30,13 +30,14 @@ import logging import os import sys -from typing import Dict, Type +from typing import Dict, Type, Callable from powerapi.actor import Actor from powerapi.database.influxdb2 import InfluxDB2 from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ - DatabaseNameAlreadyUsed + DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed from powerapi.filter import Filter +from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor from powerapi.report import HWPCReport, PowerReport, ControlReport, ProcfsReport, Report, FormulaReport from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, DirectPrometheusDB, \ VirtioFSDB, FileDB @@ -52,7 +53,12 @@ COMPONENT_DB_NAME_KEY = 'db' COMPONENT_DB_COLLECTION_KEY = 'collection' COMPONENT_DB_MANAGER_KEY = 'db_manager' -COMPONENT_DB_MAX_BUFFER_SIZE = 'max_buffer_size' +COMPONENT_DB_MAX_BUFFER_SIZE_KEY = 'max_buffer_size' +COMPONENT_URI_KEY = 'uri' + +ACTOR_NAME_KEY = 'actor_name' +TARGET_ACTORS_KEY = 'target_actors' +REGEXP_KEY = 'regexp' GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' @@ -301,7 +307,7 @@ def _actor_factory(self, actor_name: str, main_config: dict, component_config: d if 'max_buffer_size' in component_config.keys(): return PusherActor(name=actor_name, report_model=component_config[COMPONENT_MODEL_KEY], database=component_config[COMPONENT_DB_MANAGER_KEY], - max_size=component_config[COMPONENT_DB_MAX_BUFFER_SIZE]) + max_size=component_config[COMPONENT_DB_MAX_BUFFER_SIZE_KEY]) return PusherActor(name=actor_name, report_model=component_config[COMPONENT_MODEL_KEY], database=component_config[COMPONENT_DB_MANAGER_KEY], @@ -340,3 +346,51 @@ def generate(self, config: dict): report_modifier = self.factory[report_modifier_name](config['report_modifier'][report_modifier_name]) report_modifier_list.append(report_modifier) return report_modifier_list + + +class ProcessorGenerator(Generator): + """ + Generator that initialises the processor from config + """ + + def __init__(self): + Generator.__init__(self, component_group_name='processor') + + self.processor_factory = { + 'libvirt': lambda processor_config: LibvirtProcessorActor(name=processor_config[ACTOR_NAME_KEY], + uri=processor_config[COMPONENT_URI_KEY], + regexp=processor_config[REGEXP_KEY]) + + } + + def remove_processor_factory(self, processor_type: str): + """ + remove a processor from generator + """ + if processor_type not in self.processor_factory: + raise ProcessorTypeDoesNotExist(processor_type=processor_type) + del self.processor_factory[processor_type] + + def add_processor_factory(self, processor_type: str, processor_factory_function: Callable): + """ + add a processor to generator + """ + if processor_type in self.processor_factory: + raise ProcessorTypeAlreadyUsed(processor_type=processor_type) + self.processor_factory[processor_type] = processor_factory_function + + def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str): + + return self._actor_factory(actor_name, main_config, component_config) + + def _actor_factory(self, actor_name: str, _, component_config: dict): + + processor_actor_type = component_config[COMPONENT_TYPE_KEY] + + if processor_actor_type not in self.processor_factory: + msg = 'Configuration error : processor actor type ' + processor_actor_type + ' unknown' + print(msg, file=sys.stderr) + raise PowerAPIException(msg) + else: + component_config[ACTOR_NAME_KEY] = actor_name + return self.processor_factory[processor_actor_type](component_config) diff --git a/powerapi/dispatcher/dispatcher_actor.py b/powerapi/dispatcher/dispatcher_actor.py index 8443fa68..fcca6b03 100644 --- a/powerapi/dispatcher/dispatcher_actor.py +++ b/powerapi/dispatcher/dispatcher_actor.py @@ -140,6 +140,13 @@ def get_all_formula(self): """ return self.formula_dict.items() + def set_formula_factory(self, formula_factory: Callable): + """ + Set the formula_factory function + :param Callable formula_factory: The new formula_factory + """ + self.formula_factory = formula_factory + class DispatcherActor(Actor): """ @@ -164,6 +171,8 @@ def __init__(self, name: str, formula_init_function: Callable, pushers: [], rout # (func): Function for creating Formula self.formula_init_function = formula_init_function + self.pushers = pushers + # (powerapi.DispatcherState): Actor state self.state = DispatcherState(self, self._create_factory(pushers), route_table) @@ -195,3 +204,9 @@ def factory(formula_id): return formula return factory + + def update_state_formula_factory(self): + """ + Update the formula_factory function of the state by using the pusher list + """ + self.state.set_formula_factory(self._create_factory(self.pushers)) diff --git a/powerapi/exception.py b/powerapi/exception.py index 3a123675..e7726b5b 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -279,3 +279,43 @@ def __init__(self, existing_prefix: str, new_prefix: str): self.existing_prefix = existing_prefix self.msg = "The new prefix " + self.new_prefix + " has a conflict with the existing prefix " \ + self.existing_prefix + + +class LibvirtException(PowerAPIException): + """ + Exception raised when there are issues regarding the import of LibvirtException + """ + def __init__(self, _): + PowerAPIException.__init__(self) + + +class ProcessorTypeDoesNotExist(PowerAPIException): + """ + Exception raised when attempting to remove to a ProcessorActorGenerator a processor factory with a type that is not + bound to a processor factory + """ + + def __init__(self, processor_type: str): + PowerAPIException.__init__(self) + self.processor_type = processor_type + + +class ProcessorTypeAlreadyUsed(PowerAPIException): + """ + Exception raised when attempting to add to a ProcessorActorGenerator a processor factory with a type already bound + to another processor factory + """ + + def __init__(self, processor_type: str): + PowerAPIException.__init__(self) + self.processor_type = processor_type + + +class UnsupportedActorTypeException(ParserException): + """ + Exception raised when the binding manager do not support an actor type + """ + + def __init__(self, actor_type: str): + ParserException.__init__(self, argument_name=actor_type) + self.msg = 'Unsupported Actor Type ' + actor_type diff --git a/powerapi/processor/__init__.py b/powerapi/processor/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/handlers.py b/powerapi/processor/handlers.py new file mode 100644 index 00000000..7c477cb3 --- /dev/null +++ b/powerapi/processor/handlers.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from powerapi.handler import InitHandler +from powerapi.report import Report + + +class ReportHandler(InitHandler): + """ + Put the received report in a buffer + + the buffer is empty every *delay* ms or if its size exceed *max_size* + + :param int delay: number of ms before message containing in the buffer will be writen in database + :param int max_size: maximum of message that the buffer can store before write them in database + """ + + def handle(self, report): + """ + Modify the report + + :param Report report: Report to be modified + """ + self._process_message(report=report) + + def _process_message(self, report: Report): + """ + Method to be implemented to modify the report + :param Report report: Report to be modified + """ + + def _send_report(self, report: Report): + """ + Send the report to the actor target + """ + for target in self.state.target_actors: + target.send_data(report) diff --git a/powerapi/processor/k8s/__init__.py b/powerapi/processor/k8s/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/k8s/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/libvirt/__init__.py b/powerapi/processor/libvirt/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/libvirt/libvirt_processor_actor.py b/powerapi/processor/libvirt/libvirt_processor_actor.py new file mode 100644 index 00000000..4d4e9801 --- /dev/null +++ b/powerapi/processor/libvirt/libvirt_processor_actor.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import re + +from powerapi.exception import LibvirtException +from powerapi.message import StartMessage +from powerapi.processor.libvirt.libvirt_processor_handlers import LibvirtProcessorReportHandler, \ + LibvirtProcessorStartHandler +from powerapi.report import Report +from powerapi.actor import Actor +from powerapi.processor.processor_actor import ProcessorActor, ProcessorState + +try: + from libvirt import openReadOnly +except ImportError: + logging.getLogger().info("libvirt-python is not installed.") + + libvirtError = LibvirtException + openReadOnly = None + + +class LibvirtProcessorState(ProcessorState): + """ + + """ + + def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors) + self.regexp = re.compile(regexp) + self.daemon_uri = None if uri == '' else uri + print('used openReadOnly', str(type(openReadOnly))) + self.libvirt = openReadOnly(self.daemon_uri) + + +class LibvirtProcessorActor(ProcessorActor): + """ + Processor Actor that modifies reports by replacing libvirt id by open stak uuid + """ + + def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, + level_logger: int = logging.WARNING, + timeout: int = 5000): + ProcessorActor.__init__(self, name=name, target_actors=target_actors, level_logger=level_logger, + timeout=timeout) + self.state = LibvirtProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors) + + def setup(self): + """ + Define ReportMessage handler and StartMessage handler + """ + ProcessorActor.setup(self) + self.add_handler(message_type=StartMessage, handler=LibvirtProcessorStartHandler(state=self.state)) + self.add_handler(message_type=Report, handler=LibvirtProcessorReportHandler(state=self.state)) diff --git a/powerapi/processor/libvirt/libvirt_processor_handlers.py b/powerapi/processor/libvirt/libvirt_processor_handlers.py new file mode 100644 index 00000000..d89f1570 --- /dev/null +++ b/powerapi/processor/libvirt/libvirt_processor_handlers.py @@ -0,0 +1,82 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import re + +from powerapi.actor import State +from powerapi.exception import LibvirtException +from powerapi.processor.handlers import ReportHandler +from powerapi.handler import StartHandler +from powerapi.report import Report + +try: + from libvirt import libvirtError +except ImportError: + logging.getLogger().info("libvirt-python is not installed.") + + libvirtError = LibvirtException + + +class LibvirtProcessorReportHandler(ReportHandler): + """ + Modify reports by replacing libvirt id by open stak uuid + """ + + def __init__(self, state): + ReportHandler.__init__(self, state=state) + + def handle(self, report: Report): + """ + Modify reports by replacing libvirt id by open stak uuid + + :param Report report: Report to be modified + """ + result = re.match(self.state.regexp, report.target) + if result is not None: + domain_name = result.groups(0)[0] + try: + domain = self.state.libvirt.lookupByName(domain_name) + report.metadata["domain_id"] = domain.UUIDString() + except libvirtError: + pass + + self._send_report(report=report) + + +class LibvirtProcessorStartHandler(StartHandler): + """ + Initialize the target actors + """ + + def __init__(self, state: State): + StartHandler.__init__(self, state=state) + + def initialization(self): + for actor in self.state.target_actors: + actor.connect_data() diff --git a/powerapi/processor/processor_actor.py b/powerapi/processor/processor_actor.py new file mode 100644 index 00000000..e46ac9a4 --- /dev/null +++ b/powerapi/processor/processor_actor.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging + +from powerapi.actor import State, Actor +from powerapi.handler import PoisonPillMessageHandler +from powerapi.message import PoisonPillMessage + + +class ProcessorState(State): + """ + Processor Actor State + + Contains in addition to State values : + - the targets actors + """ + + def __init__(self, actor: Actor, target_actors: list): + """ + :param list target_actors: List of target actors for the processor + """ + super().__init__(actor) + + if not target_actors: + target_actors = [] + + self.target_actors = target_actors + + +class ProcessorActor(Actor): + """ + ProcessorActor class + + A processor modifies a report and sends the modified report to a list of target + actor. + """ + + def __init__(self, name: str, target_actors: list = None, level_logger: int = logging.WARNING, timeout: int = 5000): + """ + :param list target_actors: Actors that will receive the modified report + """ + + Actor.__init__(self, name, level_logger, timeout) + + #: (State): Actor State. + self.state = ProcessorState(actor=self, target_actors=target_actors) + + def setup(self): + """ + Define StartMessage handler and PoisonPillMessage handler and ReportMessage handler + """ + self.add_handler(message_type=PoisonPillMessage, handler=PoisonPillMessageHandler(state=self.state)) + # self.add_handler(message_type=StartMessage, handler=StartHandler(state=self.state)) + + def add_target_actor(self, actor: Actor): + """ + Add the given actor to the list of targets + :param actor: Actor to be defined as target + """ + self.state.target_actors.append(actor) diff --git a/powerapi/puller/simple/simple_puller_actor.py b/powerapi/puller/simple/simple_puller_actor.py index d25c5c3f..3c5faa13 100644 --- a/powerapi/puller/simple/simple_puller_actor.py +++ b/powerapi/puller/simple/simple_puller_actor.py @@ -39,13 +39,13 @@ class SimplePullerState(State): """ - Simple Puller Actor State + Simple Puller Actor State - Contains in addition to State values : - - the number of reports to send - - the report type to send - - the report filter - """ + Contains in addition to State values : + - the number of reports to send + - the report type to send + - the report filter + """ def __init__(self, actor, number_of_reports_to_send: int, report_type_to_send: Type[Report], report_filter): """ diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 53dc8612..84419f3e 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -31,6 +31,10 @@ import pytest import tests.utils.cli as test_files_module +from powerapi.cli.binding_manager import INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, ProcessorBindingManager +from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator +from powerapi.dispatcher import DispatcherActor, RouteTable +from powerapi.filter import Filter from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ generate_cli_configuration_from_json_file from powerapi.cli.config_parser import SubgroupConfigParser, BaseConfigParser, store_true, RootConfigParser @@ -182,6 +186,23 @@ def several_inputs_outputs_stream_filedb_without_some_arguments_config(several_i return several_inputs_outputs_stream_config +@pytest.fixture +def several_libvirt_processors_config(): + """ + Configuration with several libvirt processors + """ + return load_configuration_from_json_file(file_name='several_libvirt_processors_configuration.json') + + +@pytest.fixture +def several_libvirt_processors_without_some_arguments_config(): + """ + Configuration with several libvirt processors + """ + return load_configuration_from_json_file( + file_name='several_libvirt_processors_without_some_arguments_configuration.json') + + @pytest.fixture def csv_io_postmortem_config(invalid_csv_io_stream_config): """ @@ -231,6 +252,14 @@ def config_without_output(csv_io_postmortem_config): return csv_io_postmortem_config +@pytest.fixture +def libvirt_processor_config(): + """ + Configuration with libvirt as processor + """ + return load_configuration_from_json_file(file_name='libvirt_processor_configuration.json') + + @pytest.fixture() def subgroup_parser(): """ @@ -444,6 +473,9 @@ def root_config_parsing_manager_with_mandatory_and_optional_arguments(): @pytest.fixture def test_files_path(): + """ + Return the path of directory containing tests files + """ return test_files_module.__path__[0] @@ -462,8 +494,101 @@ def cli_configuration(config_file: str): @pytest.fixture() def empty_cli_configuration(): + """ + Clean the CLI arguments + """ sys.argv = [] yield None sys.argv = [] + + +@pytest.fixture +def puller_to_libvirt_processor_binding_configuration(): + """ + Return a dictionary containing bindings with a libvirt processor + """ + return load_configuration_from_json_file(file_name='puller_to_libvirt_processor_binding_configuration.json') + + +@pytest.fixture +def pusher_to_libvirt_processor_wrong_binding_configuration(): + """ + Return a dictionary containing bindings with a libvirt processor with a bind to a pusher + """ + return load_configuration_from_json_file(file_name='pusher_to_libvirt_processor_wrong_binding_configuration.json') + + +@pytest.fixture +def non_existent_puller_to_libvirt_processor_configuration(): + """ + Return a dictionary containing bindings with a libvirt processor with a bind to a pusher + """ + return load_configuration_from_json_file( + file_name='libvirt_processor_binding_with_non_existent_puller_configuration.json') + + +@pytest.fixture +def libvirt_processor_binding_actors_and_dictionary(puller_to_libvirt_processor_binding_configuration): + """ + Return a list of dictionary which contains actors as well as the expected unified dictionary related to the actor + list + """ + actors = [] + expected_actors_dictionary = {INPUT_GROUP: {}, OUTPUT_GROUP: {}, PROCESSOR_GROUP: {}} + + report_filter = Filter() + puller_generator = PullerGenerator(report_filter=report_filter) + pullers = puller_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + actors.append(pullers) + + expected_actors_dictionary[INPUT_GROUP].update(pullers) + + pusher_generator = PusherGenerator() + pushers = pusher_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + actors.append(pushers) + + expected_actors_dictionary[OUTPUT_GROUP].update(pushers) + + route_table = RouteTable() + + dispatcher = DispatcherActor(name='dispatcher', formula_init_function=None, pushers=pushers, + route_table=route_table) + + report_filter.filter(lambda msg: True, dispatcher) + + processor_generator = ProcessorGenerator() + processors = processor_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + actors.append(processors) + + expected_actors_dictionary[PROCESSOR_GROUP].update(processors) + + return actors, expected_actors_dictionary + + +@pytest.fixture +def dispatcher_actor_in_dictionary(): + """ + Return a DistpatcherActor in a dictionary + """ + + route_table = RouteTable() + + dispatcher = DispatcherActor(name='dispatcher', formula_init_function=None, pushers=None, + route_table=route_table) + + return {dispatcher.name: dispatcher} + + +@pytest.fixture +def libvirt_processor_binding_manager(libvirt_processor_binding_actors_and_dictionary): + """ + Return a ProcessorBindingManager with a libvirt Processor + """ + actors = {} + + for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + actors.update(current_actors) + + return ProcessorBindingManager(actors=actors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py new file mode 100644 index 00000000..09db4ba6 --- /dev/null +++ b/tests/unit/cli/test_binding_manager.py @@ -0,0 +1,150 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +import pytest + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from powerapi.cli.binding_manager import ProcessorBindingManager, INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, \ + BINDING_GROUP +from powerapi.dispatcher import DispatcherActor +from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData +from powerapi.processor.processor_actor import ProcessorActor + + +def check_default_processor_binding_manager_default_actors_content(processor_manager: ProcessorBindingManager): + """ + Check the default size for actors dictionary of the manager + :param ProcessorBindingManager processor_manager: Binding manager to check the size + """ + assert len(processor_manager.actors) == 3 + assert len(processor_manager.actors[INPUT_GROUP]) == 0 + assert len(processor_manager.actors[OUTPUT_GROUP]) == 0 + assert len(processor_manager.actors[PROCESSOR_GROUP]) == 0 + + +def test_create_processor_binding_manager_with_actors(libvirt_processor_binding_actors_and_dictionary): + """ + Test that a ProcessorBindingManager is correctly created when a actor dictionary is provided + """ + actors = {} + + for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + actors.update(current_actors) + binding_manager = ProcessorBindingManager(actors=actors) + + assert binding_manager.actors == libvirt_processor_binding_actors_and_dictionary[1] + + +def test_create_processor_binding_manager_without_actors(): + """ + Test that a ProcessorBindingManager is correctly created without a dictionary + """ + binding_manager = ProcessorBindingManager(actors=None) + + check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + + +def test_create_processor_binding_manager_raise_exception_with_wrong_actor_type(dispatcher_actor_in_dictionary): + """ + Test that a ProcessorBindingManager is correctly created without a dictionary + """ + with pytest.raises(UnsupportedActorTypeException): + _ = ProcessorBindingManager(actors=dispatcher_actor_in_dictionary) + + +def test_add_actors(libvirt_processor_binding_actors_and_dictionary): + """ + Test that a dictionary is correctly generated according to a list of actors + """ + binding_manager = ProcessorBindingManager(actors=[]) + + check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + + for actors in libvirt_processor_binding_actors_and_dictionary[0]: + binding_manager.add_actors(actors) + + assert binding_manager.actors == libvirt_processor_binding_actors_and_dictionary[1] + + +def test_add_actors_raise_exception_with_wrong_actor_type(dispatcher_actor_in_dictionary): + """ + Test that a dictionary is correctly generated according to a list of actors + """ + binding_manager = ProcessorBindingManager(actors=[]) + + check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + + with pytest.raises(UnsupportedActorTypeException): + binding_manager.add_actors(dispatcher_actor_in_dictionary) + + check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + + +def test_process_bindings_for_libvirt_processor(puller_to_libvirt_processor_binding_configuration, + libvirt_processor_binding_actors_and_dictionary): + """ + Test that the bindings between a puller and a processor are correctly created + """ + actors = {} + + for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + actors.update(current_actors) + + binding_manager = ProcessorBindingManager(actors=actors) + + assert len(actors['one_puller'].state.report_filter.filters) == 1 + assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], DispatcherActor) + + binding_manager.process_bindings(bindings=puller_to_libvirt_processor_binding_configuration[BINDING_GROUP]) + + assert len(actors['one_puller'].state.report_filter.filters) == 1 + assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], ProcessorActor) + assert actors['one_puller'].state.report_filter.filters[0][1] == actors['my_processor'] + + +def test_process_bindings_for_libvirt_processor_raise_exception_with_wrong_binding_types( + pusher_to_libvirt_processor_wrong_binding_configuration, + libvirt_processor_binding_manager): + """ + Test that an exception is raised with a wrong type for the from actor in a binding + """ + + with pytest.raises(BadInputData): + libvirt_processor_binding_manager.process_bindings( + bindings=pusher_to_libvirt_processor_wrong_binding_configuration[BINDING_GROUP]) + + +def test_process_bindings_for_libvirt_processor_raisse_exception_with_non_existent_puller( + non_existent_puller_to_libvirt_processor_configuration, + libvirt_processor_binding_manager): + """ + Test that an exception is raised with a non-existent puller + """ + + with pytest.raises(BadInputData): + libvirt_processor_binding_manager.process_bindings( + bindings=non_existent_puller_to_libvirt_processor_configuration[BINDING_GROUP]) diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 3a15b757..394203d0 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -1,8 +1,6 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille # All rights reserved. -import os -import re # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -28,10 +26,17 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re +from re import Pattern + import pytest -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator +from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist +from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor +from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor from powerapi.database import MongoDB, CsvDB, SocketDB, InfluxDB, PrometheusDB, InfluxDB2 @@ -110,7 +115,8 @@ def test_generate_several_pullers_from_config(several_inputs_outputs_stream_conf assert False -def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input(several_inputs_outputs_stream_mongo_without_some_arguments_config): +def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input( + several_inputs_outputs_stream_mongo_without_some_arguments_config): """ Test that PullerGenerator raise a PowerAPIException when some arguments are missing for mongo input """ @@ -120,7 +126,8 @@ def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input(s generator.generate(several_inputs_outputs_stream_mongo_without_some_arguments_config) -def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_actors(several_inputs_outputs_stream_csv_without_some_arguments_config): +def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_actors( + several_inputs_outputs_stream_csv_without_some_arguments_config): """ Test that PullerGenerator generates the csv related actors even if there are some missing arguments """ @@ -130,7 +137,8 @@ def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_ac assert len(pullers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['input']) - for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input'].items(): + for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input'].\ + items(): if current_puller_infos['type'] == 'csv': assert puller_name in pullers @@ -171,7 +179,8 @@ def test_remove_model_factory_that_does_not_exist_on_a_DBActorGenerator_must_rai assert len(generator.report_classes) == 5 -def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception(mongodb_input_output_stream_config): +def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception( + mongodb_input_output_stream_config): """ Test that PullGenerator raises PowerAPIException when the model class is not defined """ @@ -181,7 +190,8 @@ def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_rep _ = generator.generate(mongodb_input_output_stream_config) -def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_input_must_call_sys_exit_(mongodb_input_output_stream_config): +def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_input_must_call_sys_exit_( + mongodb_input_output_stream_config): """ Test that PullGenerator raises a PowerAPIException when an input type is not defined """ @@ -291,7 +301,8 @@ def test_generate_several_pushers_from_config(several_inputs_outputs_stream_conf assert False -def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output(several_inputs_outputs_stream_mongo_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output( + several_inputs_outputs_stream_mongo_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for mongo output """ @@ -301,7 +312,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_mongo_output( generator.generate(several_inputs_outputs_stream_mongo_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output(several_inputs_outputs_stream_influx_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output( + several_inputs_outputs_stream_influx_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for influx output """ @@ -311,7 +323,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_influx_output generator.generate(several_inputs_outputs_stream_influx_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_output(several_inputs_outputs_stream_prometheus_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_output( + several_inputs_outputs_stream_prometheus_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for prometheus output """ @@ -321,7 +334,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_prometheus_ou generator.generate(several_inputs_outputs_stream_prometheus_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_output(several_inputs_outputs_stream_opentsdb_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_output( + several_inputs_outputs_stream_opentsdb_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for opentsdb output """ @@ -331,7 +345,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_opentsdb_outp generator.generate(several_inputs_outputs_stream_opentsdb_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_output(several_inputs_outputs_stream_virtiofs_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_output( + several_inputs_outputs_stream_virtiofs_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for virtiofs output """ @@ -341,7 +356,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_virtiofs_outp generator.generate(several_inputs_outputs_stream_virtiofs_without_some_arguments_config) -def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output(several_inputs_outputs_stream_filedb_without_some_arguments_config): +def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output( + several_inputs_outputs_stream_filedb_without_some_arguments_config): """ Test that PusherGenerator raises a PowerAPIException when some arguments are missing for filedb output """ @@ -351,7 +367,8 @@ def test_generate_pusher_raise_exception_when_missing_arguments_in_filedb_output generator.generate(several_inputs_outputs_stream_filedb_without_some_arguments_config) -def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_actors(several_inputs_outputs_stream_csv_without_some_arguments_config): +def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_actors( + several_inputs_outputs_stream_csv_without_some_arguments_config): """ Test that PusherGenerator generates the csv related actors even if there are some missing arguments """ @@ -362,7 +379,8 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a assert len(pushers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['output']) - for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output'].items(): + for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output'].\ + items(): pusher_type = current_pusher_infos['type'] if pusher_type == 'csv': assert pusher_name in pushers @@ -376,3 +394,64 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a generation_checked = True assert generation_checked + + +################################ +# ProcessorActorGenerator Test # +################################ +def test_generate_processor_from_empty_config_dict_raise_an_exception(): + """ + Test that ProcessGenerator raise an exception when there is no processor argument + """ + conf = {} + generator = ProcessorGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(conf) + + +def test_generate_processor_from_libvirt_config(libvirt_processor_config): + """ + Test that generation for libvirt processor from a config works correctly + """ + generator = ProcessorGenerator() + + processors = generator.generate(libvirt_processor_config) + + assert len(processors) == len(libvirt_processor_config) + assert 'my_processor' in processors + processor = processors['my_processor'] + + assert isinstance(processor, LibvirtProcessorActor) + + assert processor.state.daemon_uri is None + assert isinstance(processor.state.regexp, Pattern) + + +def test_generate_several_libvirt_processors_from_config(several_libvirt_processors_config): + """ + Test that several libvirt processors are correctly generated + """ + generator = ProcessorGenerator() + + processors = generator.generate(several_libvirt_processors_config) + + assert len(processors) == len(several_libvirt_processors_config['processor']) + + for processor_name, current_processor_infos in several_libvirt_processors_config['processor'].items(): + assert processor_name in processors + assert isinstance(processors[processor_name], LibvirtProcessorActor) + + assert processors[processor_name].state.daemon_uri is None + assert isinstance(processors[processor_name].state.regexp, Pattern) + + +def test_generate_libvirt_processor_raise_exception_when_missing_arguments( + several_libvirt_processors_without_some_arguments_config): + """ + Test that ProcessorGenerator raises a PowerAPIException when some arguments are missing for libvirt processor + """ + generator = ProcessorGenerator() + + with pytest.raises(PowerAPIException): + generator.generate(several_libvirt_processors_without_some_arguments_config) diff --git a/tests/unit/processor/__init__.py b/tests/unit/processor/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/libvirt/__init__.py b/tests/unit/processor/libvirt/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/libvirt/test_libvirt_processor.py b/tests/unit/processor/libvirt/test_libvirt_processor.py new file mode 100644 index 00000000..2d7d9d41 --- /dev/null +++ b/tests/unit/processor/libvirt/test_libvirt_processor.py @@ -0,0 +1,99 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=no-self-use,arguments-differ,unused-argument + +from time import sleep +import pytest +from mock.mock import patch + +from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor +from powerapi.report import Report +from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe +from tests.utils.actor.dummy_actor import DummyActor +from tests.utils.libvirt import REGEXP, LIBVIRT_TARGET_NAME1, UUID_1, MockedLibvirt, LIBVIRT_TARGET_NAME2 + + +BAD_TARGET = 'lkjqlskjdlqksjdlkj' +DISPATCHER_NAME = 'test_libvirt_processor_dispatcher' +REPORT_TYPE_TO_BE_SENT = Report + + +class TestLibvirtProcessor(AbstractTestActor): + """ + Class to test the processor related to libvirt + """ + @pytest.fixture + def started_fake_dispatcher(self, dummy_pipe_in): + """ + Return a started DummyActor. When the test is finished, the actor is stopped + """ + dispatcher = DummyActor(name=DISPATCHER_NAME, pipe=dummy_pipe_in, message_type=REPORT_TYPE_TO_BE_SENT) + dispatcher.start() + + yield dispatcher + if dispatcher.is_alive(): + dispatcher.terminate() + + @pytest.fixture + def actor(self, started_fake_dispatcher): + with patch('powerapi.processor.libvirt.libvirt_processor_actor.openReadOnly', return_value=MockedLibvirt()): + return LibvirtProcessorActor(name='processor_actor', uri='', regexp=REGEXP, + target_actors=[started_fake_dispatcher]) + + def test_modify_report_that_not_match_regexp_must_not_modify_report(self, started_actor, + dummy_pipe_out, + shutdown_system): + """ + Test that te LibvirtProcessorActor does not modify an report that does not match the regexp + """ + report = Report(0, 'sensor', BAD_TARGET) + started_actor.send_data(msg=report) + sleep(1) + assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) + + def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): + """ + Test that a report matching the regexp of the processor is actually modified + """ + report = Report(0, 'sensor', LIBVIRT_TARGET_NAME1) + started_actor.send_data(msg=report) + new_report = recv_from_pipe(dummy_pipe_out, 2)[1] + assert new_report.metadata["domain_id"] == UUID_1 + + def test_modify_report_that_match_regexp_but_with_wrong_domain_name_must_not_modify_report(self, started_actor, + dummy_pipe_out, + shutdown_system): + """ + Test that a report matching the regexp but with wrong domain name is not modified by the processor + """ + report = Report(0, 'sensor', LIBVIRT_TARGET_NAME2) + started_actor.send_data(msg=report) + sleep(1) + assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) diff --git a/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json new file mode 100644 index 00000000..77a6951d --- /dev/null +++ b/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json @@ -0,0 +1,35 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp" + } + }, + "binding": { + "b1": { + "from": "input.non_existent_puller", + "to": "processor.my_processor" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_processor_configuration.json b/tests/utils/cli/libvirt_processor_configuration.json new file mode 100644 index 00000000..d81f5867 --- /dev/null +++ b/tests/utils/cli/libvirt_processor_configuration.json @@ -0,0 +1,9 @@ +{ + "processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json b/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json new file mode 100644 index 00000000..8aceb3bc --- /dev/null +++ b/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json @@ -0,0 +1,35 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp" + } + }, + "binding": { + "b1": { + "from": "input.one_puller", + "to": "processor.my_processor" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json b/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json new file mode 100644 index 00000000..3145423c --- /dev/null +++ b/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json @@ -0,0 +1,35 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp" + } + }, + "binding": { + "b1": { + "from": "output.one_pusher", + "to": "processor.my_processor" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_libvirt_processors_configuration.json b/tests/utils/cli/several_libvirt_processors_configuration.json new file mode 100644 index 00000000..7403a540 --- /dev/null +++ b/tests/utils/cli/several_libvirt_processors_configuration.json @@ -0,0 +1,34 @@ +{ + "processor": { + "my_processor_1": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_1" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_2" + }, + "my_processor_3": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_3" + }, + "my_processor_4": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_4" + }, + "my_processor_5": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_5" + }, + "my_processor_6": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json new file mode 100644 index 00000000..a86bdec6 --- /dev/null +++ b/tests/utils/cli/several_libvirt_processors_without_some_arguments_configuration.json @@ -0,0 +1,32 @@ +{ + "processor": { + "my_processor_1": { + "type": "libvirt", + "regexp": "a_reg_exp_1" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "" + }, + "my_processor_3": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_3" + }, + "my_processor_4": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_4" + }, + "my_processor_5": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_5" + }, + "my_processor_6": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp_6" + } + } +} \ No newline at end of file From f5f5acb0c025cec2c1a9daecaee92bffcaa8227b Mon Sep 17 00:00:00 2001 From: daniel Date: Wed, 26 Jul 2023 16:29:21 +0200 Subject: [PATCH 02/35] refactor: Clean code --- tests/unit/actor/abstract_test_actor.py | 2 -- .../puller/simple/test_simple_puller_actor.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/unit/actor/abstract_test_actor.py b/tests/unit/actor/abstract_test_actor.py index 081d2366..b706ff0b 100644 --- a/tests/unit/actor/abstract_test_actor.py +++ b/tests/unit/actor/abstract_test_actor.py @@ -316,8 +316,6 @@ def test_send_PoisonPillMessage_set_actor_alive_to_False(self, init_actor): def test_send_StartMessage_answer_OkMessage(self, init_actor): init_actor.send_control(StartMessage(SENDER_NAME)) msg = init_actor.receive_control(2000) - print('Message....') - print(msg) assert isinstance(msg, OKMessage) def test_send_StartMessage_to_already_started_actor_answer_ErrorMessage(self, started_actor): diff --git a/tests/unit/puller/simple/test_simple_puller_actor.py b/tests/unit/puller/simple/test_simple_puller_actor.py index 0ddd2f8f..e9eb2f48 100644 --- a/tests/unit/puller/simple/test_simple_puller_actor.py +++ b/tests/unit/puller/simple/test_simple_puller_actor.py @@ -64,7 +64,7 @@ class TestSimplePuller(AbstractTestActor): @pytest.fixture def started_fake_dispatcher(self, dummy_pipe_in): """ - Return a started DummyActor. When the test is finished, the actor is stopped + Return a started DummyActor. When the test is finished, the actor is stopped """ dispatcher = DummyActor(DISPATCHER_NAME, dummy_pipe_in, REPORT_TYPE_TO_BE_SENT) dispatcher.start() @@ -75,7 +75,7 @@ def started_fake_dispatcher(self, dummy_pipe_in): @pytest.fixture def fake_filter(self, started_fake_dispatcher): """ - Return a fake filter for a started dispatcher. The use rule always returns True + Return a fake filter for a started dispatcher. The use rule always returns True """ fake_filter = Filter() fake_filter.filter(filter_rule, started_fake_dispatcher) @@ -84,7 +84,7 @@ def fake_filter(self, started_fake_dispatcher): @pytest.fixture def empty_filter(self): """ - Return a filter withour rules + Return a filter withour rules """ fake_filter = Filter() return fake_filter @@ -97,7 +97,7 @@ def actor(self, fake_filter): @pytest.fixture def actor_without_rules(self, empty_filter): """ - Return a SimplePullerActor with a empty filter + Return a SimplePullerActor with a empty filter """ return SimplePullerActor(name=ACTOR_NAME, number_of_reports_to_send=NUMBER_OF_REPORTS_TO_SEND, report_type_to_send=REPORT_TYPE_TO_BE_SENT, report_filter=empty_filter) @@ -105,8 +105,8 @@ def actor_without_rules(self, empty_filter): @pytest.fixture def init_actor_without_rules(self, actor_without_rules): """ - Return an initialized actor, i.e., started and with data and control sockets connected. At the end of the - test, the actor is stopped + Return an initialized actor, i.e., started and with data and control sockets connected. At the end of the + test, the actor is stopped """ actor_without_rules.start() actor_without_rules.connect_data() @@ -119,7 +119,7 @@ def init_actor_without_rules(self, actor_without_rules): @pytest.fixture def init_actor_without_terminate(self, actor): """ - Return an initialized actor, i.e., started and with data and control sockets connected + Return an initialized actor, i.e., started and with data and control sockets connected """ actor.start() actor.connect_data() @@ -128,7 +128,7 @@ def init_actor_without_terminate(self, actor): def test_create_simple_puller_without_rules_is_no_initialized(self, init_actor_without_rules, shutdown_system): """ - Check that a SimplePuller without rules is no initialized + Check that a SimplePuller without rules is no initialized """ init_actor_without_rules.send_control(StartMessage('system')) @@ -141,7 +141,7 @@ def test_start_actor_send_reports_to_dispatcher(self, shutdown_system): """ - Check that a SimplePuller sends reports to dispatcher + Check that a SimplePuller sends reports to dispatcher """ count = 0 report = REPORT_TYPE_TO_BE_SENT.create_empty_report() @@ -158,7 +158,7 @@ def test_starting_actor_terminate_itself_after_poison_message_reception(self, in shutdown_system): """ - Check that a SimplePuller stops when it receives a PoisonPillMessage + Check that a SimplePuller stops when it receives a PoisonPillMessage """ init_actor_without_terminate.send_control(PoisonPillMessage('simple-test-simple-puller')) assert not is_actor_alive(init_actor_without_terminate) From 7b5f8670873cb00dd5ec894487cc185bd2a339f4 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 27 Jul 2023 10:14:06 +0200 Subject: [PATCH 03/35] refactor: Move UnknownMessageTypeException to exception.py --- powerapi/actor/actor.py | 5 +- powerapi/actor/state.py | 2 +- .../simple/simple_dispatcher_handlers.py | 3 +- powerapi/exception.py | 6 + powerapi/handler/handler.py | 4 +- .../handler/poison_pill_message_handler.py | 3 +- powerapi/message.py | 5 - powerapi/processor/k8s/k8s_processor_actor.py | 146 ++++++++++++++++++ .../processor/k8s/k8s_processor_handlers.py | 28 ++++ .../libvirt/libvirt_processor_actor.py | 2 +- .../puller/simple/simple_puller_handlers.py | 3 +- 11 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 powerapi/processor/k8s/k8s_processor_actor.py create mode 100644 powerapi/processor/k8s/k8s_processor_handlers.py diff --git a/powerapi/actor/actor.py b/powerapi/actor/actor.py index 74f96172..ea1a70cb 100644 --- a/powerapi/actor/actor.py +++ b/powerapi/actor/actor.py @@ -36,9 +36,8 @@ import traceback import setproctitle -from powerapi.exception import PowerAPIExceptionWithMessage +from powerapi.exception import PowerAPIExceptionWithMessage, UnknownMessageTypeException from powerapi.message import PoisonPillMessage -from powerapi.message import UnknownMessageTypeException from powerapi.handler import HandlerException from .socket_interface import SocketInterface @@ -229,7 +228,7 @@ def _initial_behaviour(self): handler = self.state.get_corresponding_handler(msg) handler.handle_message(msg) except UnknownMessageTypeException: - self.logger.warning("UnknowMessageTypeException: " + str(msg)) + self.logger.warning("UnknownMessageTypeException: " + str(msg)) except HandlerException: self.logger.warning("HandlerException") diff --git a/powerapi/actor/state.py b/powerapi/actor/state.py index 7c2f16e4..eb1b322c 100644 --- a/powerapi/actor/state.py +++ b/powerapi/actor/state.py @@ -27,7 +27,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.message import UnknownMessageTypeException +from powerapi.exception import UnknownMessageTypeException from powerapi.actor.supervisor import Supervisor diff --git a/powerapi/dispatcher/simple/simple_dispatcher_handlers.py b/powerapi/dispatcher/simple/simple_dispatcher_handlers.py index 0ff371e2..a9feceb8 100644 --- a/powerapi/dispatcher/simple/simple_dispatcher_handlers.py +++ b/powerapi/dispatcher/simple/simple_dispatcher_handlers.py @@ -28,7 +28,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from powerapi.actor import State from powerapi.handler import StartHandler, PoisonPillMessageHandler, Handler -from powerapi.message import StartMessage, UnknownMessageTypeException +from powerapi.message import StartMessage +from powerapi.exception import UnknownMessageTypeException from powerapi.report import Report diff --git a/powerapi/exception.py b/powerapi/exception.py index e7726b5b..de52bb8f 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -319,3 +319,9 @@ class UnsupportedActorTypeException(ParserException): def __init__(self, actor_type: str): ParserException.__init__(self, argument_name=actor_type) self.msg = 'Unsupported Actor Type ' + actor_type + + +class UnknownMessageTypeException(PowerAPIException): + """ + Exception happen when we don't know the message type + """ diff --git a/powerapi/handler/handler.py b/powerapi/handler/handler.py index 47e94e9e..c0e861d5 100644 --- a/powerapi/handler/handler.py +++ b/powerapi/handler/handler.py @@ -27,8 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.exception import PowerAPIException -from powerapi.message import Message, UnknownMessageTypeException +from powerapi.exception import PowerAPIException, UnknownMessageTypeException +from powerapi.message import Message class HandlerException(PowerAPIException): diff --git a/powerapi/handler/poison_pill_message_handler.py b/powerapi/handler/poison_pill_message_handler.py index 2e5c657f..62395763 100644 --- a/powerapi/handler/poison_pill_message_handler.py +++ b/powerapi/handler/poison_pill_message_handler.py @@ -27,7 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.message import UnknownMessageTypeException, PoisonPillMessage +from powerapi.message import PoisonPillMessage +from powerapi.exception import UnknownMessageTypeException from .handler import Handler diff --git a/powerapi/message.py b/powerapi/message.py index 9ccaef1b..2d8b520a 100644 --- a/powerapi/message.py +++ b/powerapi/message.py @@ -171,8 +171,3 @@ def __eq__(self, other): return other.is_soft == self.is_soft and other.is_hard == self.is_hard return False - -class UnknownMessageTypeException(PowerAPIException): - """ - Exception happen when we don't know the message type - """ diff --git a/powerapi/processor/k8s/k8s_processor_actor.py b/powerapi/processor/k8s/k8s_processor_actor.py new file mode 100644 index 00000000..5b019791 --- /dev/null +++ b/powerapi/processor/k8s/k8s_processor_actor.py @@ -0,0 +1,146 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from powerapi.actor import Actor +from powerapi.processor.processor_actor import ProcessorState + + +class K8sMetadataCache: + """ + K8sMetadataCache maintains a cache of pods' metadata + (namespace, labels and id of associated containers) + """ + + def __init__(self): + self.pod_labels = dict() # (ns, pod) => [labels] + self.containers_pod = dict() # container_id => (ns, pod) + self.pod_containers = dict() # (ns, pod) => [container_ids] + + def update_cache(self, message: K8sPodUpdateMessage): + """ + Update the local cache for pods. + + Register this function as a callback for K8sMonitorAgent messages. + """ + if message.event == "ADDED": + self.pod_labels[(message.namespace, message.pod)] = message.labels + self.pod_containers[ + (message.namespace, message.pod) + ] = message.containers_id + for container_id in message.containers_id: + self.containers_pod[container_id] = \ + (message.namespace, message.pod) + # logger.debug( + # "Pod added %s %s - mdt: %s", + # message.namespace, message.pod, message.containers_id + # ) + + elif message.event == "DELETED": + self.pod_labels.pop((message.namespace, message.pod), None) + for container_id in self.pod_containers.pop( + (message.namespace, message.pod), [] + ): + self.containers_pod.pop(container_id, None) + # logger.debug("Pod removed %s %s", message.namespace, message.pod) + + elif message.event == "MODIFIED": + self.pod_labels[(message.namespace, message.pod)] = message.labels + for prev_container_id in self.pod_containers.pop( + (message.namespace, message.pod), [] + ): + self.containers_pod.pop(prev_container_id, None) + self.pod_containers[ + (message.namespace, message.pod) + ] = message.containers_id + for container_id in message.containers_id: + self.containers_pod[container_id] = \ + (message.namespace, message.pod) + + # logger.debug( + # "Pod modified %s %s , mdt: %s", + # message.namespace, message.pod, message.containers_id + # ) + else: + logger.error("Error : unknown event type %s ", message.event) + + def get_container_pod(self, container_id) -> Tuple[str, str]: + """ + Get the pod for a container_id. + + :param container_id + :return a tuple (namespace, pod_name) of (None, None) if no pod + could be found for this container + """ + ns_pod = self.containers_pod.get(container_id, None) + if ns_pod is None: + return None, None + return ns_pod + + def get_pod_labels(self, namespace: str, pod_name: str) -> Dict[str, str]: + """ + Get labels for a pod. + + :param namespace + :param + :return a dict {label_name, label_value} + """ + return self.pod_labels.get((namespace, pod_name), dict) + + +class K8sProcessorState(ProcessorState): + """ + State related to a K8SProcessorActor + """ + + def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors) + self.regexp = re.compile(regexp) + self.daemon_uri = None if uri == '' else uri + print('used openReadOnly', str(type(openReadOnly))) + self.libvirt = openReadOnly(self.daemon_uri) + + +class K8sProcessorActor(ProcessorActor): + """ + Processor Actor that modifies reports by replacing libvirt id by open stak uuid + """ + + def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, + level_logger: int = logging.WARNING, + timeout: int = 5000): + ProcessorActor.__init__(self, name=name, target_actors=target_actors, level_logger=level_logger, + timeout=timeout) + self.state = LibvirtProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors) + + def setup(self): + """ + Define ReportMessage handler and StartMessage handler + """ + ProcessorActor.setup(self) + self.add_handler(message_type=StartMessage, handler=LibvirtProcessorStartHandler(state=self.state)) + self.add_handler(message_type=Report, handler=LibvirtProcessorReportHandler(state=self.state)) diff --git a/powerapi/processor/k8s/k8s_processor_handlers.py b/powerapi/processor/k8s/k8s_processor_handlers.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/k8s/k8s_processor_handlers.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/libvirt/libvirt_processor_actor.py b/powerapi/processor/libvirt/libvirt_processor_actor.py index 4d4e9801..4f522863 100644 --- a/powerapi/processor/libvirt/libvirt_processor_actor.py +++ b/powerapi/processor/libvirt/libvirt_processor_actor.py @@ -48,7 +48,7 @@ class LibvirtProcessorState(ProcessorState): """ - + State related to a LibvirtProcssorActor """ def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): diff --git a/powerapi/puller/simple/simple_puller_handlers.py b/powerapi/puller/simple/simple_puller_handlers.py index 89146ffd..61dbb762 100644 --- a/powerapi/puller/simple/simple_puller_handlers.py +++ b/powerapi/puller/simple/simple_puller_handlers.py @@ -29,7 +29,8 @@ from powerapi.actor import State from powerapi.handler import Handler, StartHandler -from powerapi.message import Message, SimplePullerSendReportsMessage, UnknownMessageTypeException +from powerapi.message import Message, SimplePullerSendReportsMessage +from powerapi.exception import UnknownMessageTypeException from powerapi.puller.handlers import PullerInitializationException From a05a65004544a4809f8a596689ab2be85617dde9 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 17 Aug 2023 14:17:33 +0200 Subject: [PATCH 04/35] feat: Add K8sMonitorActor and K8sProcessorActor --- powerapi/cli/generator.py | 18 +- powerapi/message.py | 37 ++- powerapi/processor/handlers.py | 23 +- powerapi/processor/k8s/k8s_monitor_actor.py | 241 ++++++++++++++ .../processor/k8s/k8s_monitor_handlers.py | 62 ++++ powerapi/processor/k8s/k8s_processor_actor.py | 97 ++++-- .../processor/k8s/k8s_processor_handlers.py | 120 +++++++ .../libvirt/libvirt_processor_actor.py | 1 - .../libvirt/libvirt_processor_handlers.py | 6 +- powerapi/processor/processor_actor.py | 5 +- tests/unit/actor/abstract_test_actor.py | 21 ++ tests/unit/processor/conftest.py | 223 +++++++++++++ tests/unit/processor/k8s/__init__.py | 28 ++ tests/unit/processor/k8s/test_k8s_monitor.py | 121 +++++++ .../unit/processor/k8s/test_k8s_processor.py | 298 ++++++++++++++++++ tests/utils/actor/dummy_handlers.py | 2 +- 16 files changed, 1238 insertions(+), 65 deletions(-) create mode 100644 powerapi/processor/k8s/k8s_monitor_actor.py create mode 100644 powerapi/processor/k8s/k8s_monitor_handlers.py create mode 100644 tests/unit/processor/conftest.py create mode 100644 tests/unit/processor/k8s/__init__.py create mode 100644 tests/unit/processor/k8s/test_k8s_monitor.py create mode 100644 tests/unit/processor/k8s/test_k8s_processor.py diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index d3903b8d..c807160b 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -37,6 +37,8 @@ from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed from powerapi.filter import Filter +from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ + TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor from powerapi.report import HWPCReport, PowerReport, ControlReport, ProcfsReport, Report, FormulaReport from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, DirectPrometheusDB, \ @@ -59,6 +61,9 @@ ACTOR_NAME_KEY = 'actor_name' TARGET_ACTORS_KEY = 'target_actors' REGEXP_KEY = 'regexp' +K8S_API_MODE_KEY = 'ks8_api_mode' +TIME_INTERVAL_KEY = 'time_interval' +TIMEOUT_QUERY_KEY = 'time_interval' GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' @@ -359,7 +364,18 @@ def __init__(self): self.processor_factory = { 'libvirt': lambda processor_config: LibvirtProcessorActor(name=processor_config[ACTOR_NAME_KEY], uri=processor_config[COMPONENT_URI_KEY], - regexp=processor_config[REGEXP_KEY]) + regexp=processor_config[REGEXP_KEY]), + 'k8s': lambda processor_config: K8sProcessorActor(name=processor_config[ACTOR_NAME_KEY], + ks8_api_mode=None if + K8S_API_MODE_KEY not in processor_config else + processor_config[K8S_API_MODE_KEY], + time_interval=TIME_INTERVAL_DEFAULT_VALUE if + TIME_INTERVAL_KEY not in processor_config else + processor_config[TIME_INTERVAL_KEY], + timeout_query=TIMEOUT_QUERY_DEFAULT_VALUE if + TIMEOUT_QUERY_KEY not in processor_config + else processor_config[TIMEOUT_QUERY_KEY] + ) } diff --git a/powerapi/message.py b/powerapi/message.py index 2d8b520a..5952e8a9 100644 --- a/powerapi/message.py +++ b/powerapi/message.py @@ -30,7 +30,6 @@ from __future__ import annotations from typing import TYPE_CHECKING -from powerapi.exception import PowerAPIException if TYPE_CHECKING: from powerapi.database import BaseDB @@ -171,3 +170,39 @@ def __eq__(self, other): return other.is_soft == self.is_soft and other.is_hard == self.is_hard return False + +class K8sPodUpdateMessage(Message): + """ + Message sent by the K8sMonitorAgent everytime it detects a change on pod in K8S + """ + + def __init__( + self, + sender_name: str, + event: str, + namespace: str, + pod: str, + containers_id: list[str] = None, + labels: dict[str, str] = None, + ): + """ + :param str sender_name: Name of the sender + :param str event: Event name + :param str namespace: Namespace name + :param str pod: Id of the Pod + :param list containers_id: List of containers id + :param dict labels: Dictionary of labels + """ + Message.__init__(self, sender_name) + self.event = event + self.namespace = namespace + self.pod = pod + self.containers_id = containers_id if containers_id is not None else [] + self.labels = labels if labels is not None else {} + + def __str__(self): + return f"K8sPodUpdateMessage {self.event} {self.namespace} {self.pod}" + + def __eq__(self, message): + return (self.event, self.namespace, self.pod, self.containers_id, self.labels) == \ + (message.event, message.namespace, message.pod, message.containers_id, message.labels) diff --git a/powerapi/processor/handlers.py b/powerapi/processor/handlers.py index 7c477cb3..03a23c36 100644 --- a/powerapi/processor/handlers.py +++ b/powerapi/processor/handlers.py @@ -30,30 +30,11 @@ from powerapi.report import Report -class ReportHandler(InitHandler): +class ProcessorReportHandler(InitHandler): """ - Put the received report in a buffer - - the buffer is empty every *delay* ms or if its size exceed *max_size* - - :param int delay: number of ms before message containing in the buffer will be writen in database - :param int max_size: maximum of message that the buffer can store before write them in database + Processor the report by modifying it in some way and then send the modified report to targets actos """ - def handle(self, report): - """ - Modify the report - - :param Report report: Report to be modified - """ - self._process_message(report=report) - - def _process_message(self, report: Report): - """ - Method to be implemented to modify the report - :param Report report: Report to be modified - """ - def _send_report(self, report: Report): """ Send the report to the actor target diff --git a/powerapi/processor/k8s/k8s_monitor_actor.py b/powerapi/processor/k8s/k8s_monitor_actor.py new file mode 100644 index 00000000..4b16db31 --- /dev/null +++ b/powerapi/processor/k8s/k8s_monitor_actor.py @@ -0,0 +1,241 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=W0603,W0718 + +import logging +from logging import Logger +from time import sleep + +from kubernetes import client, config, watch +from kubernetes.client.configuration import Configuration +from kubernetes.client.rest import ApiException + +from powerapi.actor import State, Actor +from powerapi.message import StartMessage, PoisonPillMessage, K8sPodUpdateMessage +from powerapi.processor.k8s.k8s_monitor_handlers import K8sMonitorAgentStartMessageHandler, \ + K8sMonitorAgentPoisonPillMessageHandler + +LOCAL_CONFIG_MODE = "local" +MANUAL_CONFIG_MODE = "manual" +CLUSTER_CONFIG_MODE = "cluster" + +ADDED_EVENT = 'ADDED' +DELETED_EVENT = 'DELETED' +MODIFIED_EVENT = 'MODIFIED' + +v1_api = None + + +def local_config(): + """ + Return local kubectl + """ + config.load_kube_config() + + +def manual_config(): + """ + Return the manual configuration + """ + # Manual config + configuration = client.Configuration() + # Configure API key authorization: BearerToken + configuration.api_key["authorization"] = "YOUR_API_KEY" + # Defining host is optional and default to http://localhost + configuration.host = "http://localhost" + Configuration.set_default(configuration) + + +def cluster_config(): + """ + Return the cluster configuration + """ + config.load_incluster_config() + + +def load_k8s_client_config(logger: Logger, mode: str = None): + """ + Load K8S client configuration according to the `mode`. + If no mode is given `LOCAL_CONFIG_MODE` is used. + params: + mode : one of `LOCAL_CONFIG_MODE`, `MANUAL_CONFIG_MODE` + or `CLUSTER_CONFIG_MODE` + """ + logger.debug("Loading k8s api conf mode %s ", mode) + { + LOCAL_CONFIG_MODE: local_config, + MANUAL_CONFIG_MODE: manual_config, + CLUSTER_CONFIG_MODE: cluster_config, + }.get(mode, local_config)() + + +def get_core_v1_api(logger: Logger, mode: str = None): + """ + Returns a handler to the k8s API. + """ + global v1_api + if v1_api is None: + load_k8s_client_config(logger=logger, mode=mode) + v1_api = client.CoreV1Api() + logger.info(f"Core v1 api access : {v1_api}") + return v1_api + + +def extract_containers(pod_obj): + """ + Extract the containers ids from a pod + :param pod_obj: Pod object for extracting the containers ids + """ + if not pod_obj.status.container_statuses: + return [] + + container_ids = [] + for container_status in pod_obj.status.container_statuses: + container_id = container_status.container_id + if not container_id: + continue + # container_id actually depends on the container engine used by k8s. + # It seems that is always start with :// + # e.g. + # 'containerd://2289b494f36b93647cfefc6f6ed4d7f36161d5c2f92d1f23571878a4e85282ed' + container_id = container_id[container_id.index("//") + 2:] + container_ids.append(container_id) + + return sorted(container_ids) + + +class K8sMonitorAgentState(State): + """ + State related to a K8sMonitorAgentActor + """ + + def __init__(self, actor: Actor, time_interval: int, timeout_query: int, listener_agent: Actor, k8s_api_mode: str): + State.__init__(self, actor=actor) + self.time_interval = time_interval + self.timeout_query = timeout_query + self.listener_agent = listener_agent + self.k8s_api_mode = k8s_api_mode + self.active_monitoring = False + self.monitor_thread = None + + +class K8sMonitorAgentActor(Actor): + """ + An actor that monitors the k8s API and sends messages + when pod are created, removed or modified. + """ + + def __init__(self, name: str, listener_agent: Actor, k8s_api_mode: str = None, time_interval: int = 10, + timeout_query=5, level_logger: int = logging.WARNING): + """ + :param str name: The actor name + :param K8sProcessorActor listener_agent: actor waiting for notifications of the monitor + :param k8s_api_mode: the used k8s API mode + :param int timeout_query: Timeout for queries + :param int time_interval: Time interval for the monitoring + :pram int level_logger: The logger level + """ + Actor.__init__(self, name=name, level_logger=logging.DEBUG) + self.state = K8sMonitorAgentState(actor=self, time_interval=time_interval, timeout_query=timeout_query, + listener_agent=listener_agent, k8s_api_mode=k8s_api_mode) + + def setup(self): + """ + Define StartMessage handler and PoisonPillMessage handler + """ + print('setup monitor called') + self.add_handler(message_type=StartMessage, handler=K8sMonitorAgentStartMessageHandler(state=self.state)) + self.add_handler(message_type=PoisonPillMessage, + handler=K8sMonitorAgentPoisonPillMessageHandler(state=self.state)) + + def query_k8s(self): + """ + Query k8s for changes and send the related information to the listener + """ + while self.state.active_monitoring: + try: + self.logger.debug("Start - K8sMonitorAgentActor Querying k8s") + events = self.k8s_streaming_query(timeout_seconds=self.state.timeout_query, + k8sapi_mode=self.state.k8s_api_mode) + for event in events: + event_type, namespace, pod_name, container_ids, labels = event + self.state.listener_agent.send_data( + K8sPodUpdateMessage( + sender_name=self.name, + event=event_type, + namespace=namespace, + pod=pod_name, + containers_id=container_ids, + labels=labels + ) + ) + sleep(self.state.time_interval) + except Exception as ex: + self.logger.warning(ex) + self.logger.warning("Failed streaming query %s", ex) + + def k8s_streaming_query(self, timeout_seconds: int, k8sapi_mode: str) -> list: + """ + Return a list of events by using the provided paremeters + :param int timeout_seconds: Timeout in seconds for waiting for events + :param str k8sapi_mode: Kind of API mode + """ + api = get_core_v1_api(mode=k8sapi_mode, logger=self.logger) + events = [] + w = watch.Watch() + + try: + for event in w.stream( + api.list_pod_for_all_namespaces, timeout_seconds + ): + + if not event["type"] in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: + self.logger.warning( + "UNKNOWN EVENT TYPE : %s : %s %s", + event['type'], event['object'].metadata.name, event + ) + continue + pod_obj = event["object"] + namespace, pod_name = \ + pod_obj.metadata.namespace, pod_obj.metadata.name + container_ids = ( + [] if event["type"] == "DELETED" + else extract_containers(pod_obj) + ) + labels = pod_obj.metadata.labels + events.append( + (event["type"], namespace, pod_name, container_ids, labels) + ) + + except ApiException as ae: + self.logger.error("APIException %s %s", ae.status, ae) + except Exception as undef_e: + self.logger.error("Error when watching Exception %s %s", undef_e, event) + return events diff --git a/powerapi/processor/k8s/k8s_monitor_handlers.py b/powerapi/processor/k8s/k8s_monitor_handlers.py new file mode 100644 index 00000000..02c95f15 --- /dev/null +++ b/powerapi/processor/k8s/k8s_monitor_handlers.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from threading import Thread + +from powerapi.actor import State +from powerapi.handler import StartHandler, PoisonPillMessageHandler + + +class K8sMonitorAgentStartMessageHandler(StartHandler): + """ + Start the K8sMonitorAgent + """ + + def __init__(self, state: State): + StartHandler.__init__(self, state=state) + + def initialization(self): + self.state.active_monitoring = True + self.state.listener_agent.connect_data() + monitoring_thread = Thread(target=self.state.actor.query_k8s) + monitoring_thread.start() + self.state.monitor_thread = monitoring_thread + + +class K8sMonitorAgentPoisonPillMessageHandler(PoisonPillMessageHandler): + """ + Stop the K8sMonitorAgent + """ + + def __init__(self, state: State): + PoisonPillMessageHandler.__init__(self, state=state) + + def teardown(self, soft=False): + self.state.actor.logger.debug('teardown monitor') + self.state.active_monitoring = False + self.state.monitor_thread.join(10) diff --git a/powerapi/processor/k8s/k8s_processor_actor.py b/powerapi/processor/k8s/k8s_processor_actor.py index 5b019791..70f7e639 100644 --- a/powerapi/processor/k8s/k8s_processor_actor.py +++ b/powerapi/processor/k8s/k8s_processor_actor.py @@ -26,8 +26,30 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +This module provides two k8s specific actors +* `K8sProcessorActor`, which add k8s metadata to reports and forward them to another actor. +* `K8sMonitorAgent`, which monitors the k8s API and sends messages when pod are created, removed or modified. +""" +import logging + +from typing import Tuple, Dict + from powerapi.actor import Actor -from powerapi.processor.processor_actor import ProcessorState +from powerapi.message import K8sPodUpdateMessage, StartMessage, PoisonPillMessage +from powerapi.processor.k8s.k8s_monitor_actor import ADDED_EVENT, DELETED_EVENT, MODIFIED_EVENT +from powerapi.processor.k8s.k8s_processor_handlers import K8sProcessorActorHWPCReportHandler, \ + K8sProcessorActorK8sPodUpdateMessageHandler, K8sProcessorActorStartMessageHandler, \ + K8sProcessorActorPoisonPillMessageHandler +from powerapi.processor.processor_actor import ProcessorState, ProcessorActor +from powerapi.report import HWPCReport + +DEFAULT_K8S_CACHE_NAME = 'k8s_cache' +DEFAULT_K8S_MONITOR_NAME = 'k8s_monitor' + +TIME_INTERVAL_DEFAULT_VALUE = 10 +TIMEOUT_QUERY_DEFAULT_VALUE = 5 class K8sMetadataCache: @@ -36,10 +58,14 @@ class K8sMetadataCache: (namespace, labels and id of associated containers) """ - def __init__(self): - self.pod_labels = dict() # (ns, pod) => [labels] - self.containers_pod = dict() # container_id => (ns, pod) - self.pod_containers = dict() # (ns, pod) => [container_ids] + def __init__(self, name: str = DEFAULT_K8S_CACHE_NAME, level_logger: int = logging.WARNING): + + self.pod_labels = {} # (ns, pod) => [labels] + self.containers_pod = {} # container_id => (ns, pod) + self.pod_containers = {} # (ns, pod) => [container_ids] + + self.logger = logging.getLogger(name) + self.logger.setLevel(level_logger) def update_cache(self, message: K8sPodUpdateMessage): """ @@ -47,7 +73,7 @@ def update_cache(self, message: K8sPodUpdateMessage): Register this function as a callback for K8sMonitorAgent messages. """ - if message.event == "ADDED": + if message.event == ADDED_EVENT: self.pod_labels[(message.namespace, message.pod)] = message.labels self.pod_containers[ (message.namespace, message.pod) @@ -55,12 +81,8 @@ def update_cache(self, message: K8sPodUpdateMessage): for container_id in message.containers_id: self.containers_pod[container_id] = \ (message.namespace, message.pod) - # logger.debug( - # "Pod added %s %s - mdt: %s", - # message.namespace, message.pod, message.containers_id - # ) - elif message.event == "DELETED": + elif message.event == DELETED_EVENT: self.pod_labels.pop((message.namespace, message.pod), None) for container_id in self.pod_containers.pop( (message.namespace, message.pod), [] @@ -68,7 +90,7 @@ def update_cache(self, message: K8sPodUpdateMessage): self.containers_pod.pop(container_id, None) # logger.debug("Pod removed %s %s", message.namespace, message.pod) - elif message.event == "MODIFIED": + elif message.event == MODIFIED_EVENT: self.pod_labels[(message.namespace, message.pod)] = message.labels for prev_container_id in self.pod_containers.pop( (message.namespace, message.pod), [] @@ -81,18 +103,14 @@ def update_cache(self, message: K8sPodUpdateMessage): self.containers_pod[container_id] = \ (message.namespace, message.pod) - # logger.debug( - # "Pod modified %s %s , mdt: %s", - # message.namespace, message.pod, message.containers_id - # ) else: - logger.error("Error : unknown event type %s ", message.event) + self.logger.error("Error : unknown event type %s ", message.event) - def get_container_pod(self, container_id) -> Tuple[str, str]: + def get_container_pod(self, container_id: str) -> Tuple[str, str]: """ Get the pod for a container_id. - :param container_id + :param str container_id: Id of the container :return a tuple (namespace, pod_name) of (None, None) if no pod could be found for this container """ @@ -105,8 +123,8 @@ def get_pod_labels(self, namespace: str, pod_name: str) -> Dict[str, str]: """ Get labels for a pod. - :param namespace - :param + :param str namespace: The namespace related to the pod + :param str pod_name: The name of the pod :return a dict {label_name, label_value} """ return self.pod_labels.get((namespace, pod_name), dict) @@ -117,30 +135,41 @@ class K8sProcessorState(ProcessorState): State related to a K8SProcessorActor """ - def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): + def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, + monitor_agent_name: str, k8s_api_mode: str, time_interval: int, timeout_query: int): ProcessorState.__init__(self, actor=actor, target_actors=target_actors) - self.regexp = re.compile(regexp) - self.daemon_uri = None if uri == '' else uri - print('used openReadOnly', str(type(openReadOnly))) - self.libvirt = openReadOnly(self.daemon_uri) + self.metadata_cache = metadata_cache + self.monitor_agent = None + self.monitor_agent_name = monitor_agent_name + self.k8s_api_mode = k8s_api_mode + self.time_interval = time_interval + self.timeout_query = timeout_query class K8sProcessorActor(ProcessorActor): """ - Processor Actor that modifies reports by replacing libvirt id by open stak uuid + Processor Actor that modifies reports by adding K8s related metadata """ - def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, - level_logger: int = logging.WARNING, - timeout: int = 5000): + def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, level_logger: int = logging.WARNING, + timeout: int = 5000, time_interval: int = TIME_INTERVAL_DEFAULT_VALUE, + timeout_query: int = TIMEOUT_QUERY_DEFAULT_VALUE): ProcessorActor.__init__(self, name=name, target_actors=target_actors, level_logger=level_logger, timeout=timeout) - self.state = LibvirtProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors) + + self.state = K8sProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), + monitor_agent_name=DEFAULT_K8S_MONITOR_NAME, target_actors=target_actors, + k8s_api_mode=ks8_api_mode, time_interval=time_interval, + timeout_query=timeout_query) def setup(self): """ - Define ReportMessage handler and StartMessage handler + Define HWPCReportMessage handler, StartMessage handler and PoisonPillMessage Handler """ ProcessorActor.setup(self) - self.add_handler(message_type=StartMessage, handler=LibvirtProcessorStartHandler(state=self.state)) - self.add_handler(message_type=Report, handler=LibvirtProcessorReportHandler(state=self.state)) + self.add_handler(message_type=StartMessage, handler=K8sProcessorActorStartMessageHandler(state=self.state)) + self.add_handler(message_type=HWPCReport, handler=K8sProcessorActorHWPCReportHandler(state=self.state)) + self.add_handler(message_type=PoisonPillMessage, + handler=K8sProcessorActorPoisonPillMessageHandler(state=self.state)) + self.add_handler(message_type=K8sPodUpdateMessage, + handler=K8sProcessorActorK8sPodUpdateMessageHandler(state=self.state)) diff --git a/powerapi/processor/k8s/k8s_processor_handlers.py b/powerapi/processor/k8s/k8s_processor_handlers.py index f964fff4..e7e92354 100644 --- a/powerapi/processor/k8s/k8s_processor_handlers.py +++ b/powerapi/processor/k8s/k8s_processor_handlers.py @@ -26,3 +26,123 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from powerapi.actor import State +from powerapi.handler import Handler, StartHandler, PoisonPillMessageHandler +from powerapi.message import Message +from powerapi.processor.handlers import ProcessorReportHandler + +POD_NAMESPACE_METADATA_KEY = 'pod_namespace' +POD_NAME_METADATA_KEY = 'pod_name' + + +class K8sProcessorActorStartMessageHandler(StartHandler): + """ + Start the K8sProcessorActor + """ + + def __init__(self, state: State): + StartHandler.__init__(self, state=state) + + # + def initialization(self): + for actor in self.state.target_actors: + actor.connect_data() + + +class K8sProcessorActorHWPCReportHandler(ProcessorReportHandler): + """ + Process the HWPC Reports + """ + + def __init__(self, state: State): + ProcessorReportHandler.__init__(self, state=state) + + def handle(self, message: Message): + + # Add pod name, namespace and labels to the report + c_id = clean_up_container_id(message.target) + + print('c_id', c_id) + print('containers pods', str(self.state.metadata_cache.containers_pod)) + namespace, pod = self.state.metadata_cache.get_container_pod(c_id) + if namespace is None or pod is None: + self.state.actor.logger.warning( + f"Container with no associated pod : {message.target}, {c_id}, {namespace}, {pod}" + ) + else: + message.metadata[POD_NAMESPACE_METADATA_KEY] = namespace + message.metadata[POD_NAME_METADATA_KEY] = pod + self.state.actor.logger.debug( + f"K8sMdtModifierActor add metadata to report {c_id}, {namespace}, {pod}" + ) + + labels = self.state.metadata_cache.get_pod_labels(namespace, pod) + for label_key, label_value in labels.items(): + message.metadata[f"label_{label_key}"] = label_value + + self._send_report(report=message) + + +class K8sProcessorActorPoisonPillMessageHandler(PoisonPillMessageHandler): + """ + Stop the K8sProcessorActor + """ + + def __init__(self, state: State): + PoisonPillMessageHandler.__init__(self, state=state) + + def teardown(self, soft=False): + for actor in self.state.target_actors: + actor.close() + + +# self.state.actor.logger.debug('Killing monitor actor..') +# self.state.monitor_agent.active_monitoring = False +# self.state.monitor_agent.send_data(PoisonPillMessage(soft=soft, sender_name=self.state.actor.name)) +# if self.state.monitor_agent.is_alive(): +# self.state.monitor_agent.terminate() +# self.state.monitor_agent.socket_interface.close() +# self.state.monitor_agent.join(timeout=10) +# self.state.actor.logger.debug('teardown finished..') + + +class K8sProcessorActorK8sPodUpdateMessageHandler(Handler): + """ + Process the K8sPodUpdateMessage + """ + + def __init__(self, state: State): + Handler.__init__(self, state=state) + + def handle(self, message: Message): + self.state.actor.logger.debug(f"received K8sPodUpdateMessage message {message}") + self.state.metadata_cache.update_cache(message) + + +def clean_up_container_id(c_id): + """ + On some system, we receive a container id that requires some cleanup to match + the id returned by the k8s api + k8s creates cgroup directories, which is what we get as id from the sensor, + according to this pattern: + /kubepods//pod/ + depending on the container engine, we need to clean up the part + """ + + if "/docker-" in c_id: + # for path like : + # /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod435532e3_546d_45e2_8862_d3c7b320d2d9.slice/ + # docker-68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a.scope + # where we actually only want the end of that path : + # 68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a + try: + return c_id[c_id.rindex("/docker-") + 8: -6] + except ValueError: + return c_id + else: + # /kubepods/besteffort/pod42006d2c-cad7-4575-bfa3-91848a558743/ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5 + try: + return c_id[c_id.rindex("/") + 1:] + except ValueError: + return c_id diff --git a/powerapi/processor/libvirt/libvirt_processor_actor.py b/powerapi/processor/libvirt/libvirt_processor_actor.py index 4f522863..873ab498 100644 --- a/powerapi/processor/libvirt/libvirt_processor_actor.py +++ b/powerapi/processor/libvirt/libvirt_processor_actor.py @@ -55,7 +55,6 @@ def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): ProcessorState.__init__(self, actor=actor, target_actors=target_actors) self.regexp = re.compile(regexp) self.daemon_uri = None if uri == '' else uri - print('used openReadOnly', str(type(openReadOnly))) self.libvirt = openReadOnly(self.daemon_uri) diff --git a/powerapi/processor/libvirt/libvirt_processor_handlers.py b/powerapi/processor/libvirt/libvirt_processor_handlers.py index d89f1570..705f2532 100644 --- a/powerapi/processor/libvirt/libvirt_processor_handlers.py +++ b/powerapi/processor/libvirt/libvirt_processor_handlers.py @@ -31,7 +31,7 @@ from powerapi.actor import State from powerapi.exception import LibvirtException -from powerapi.processor.handlers import ReportHandler +from powerapi.processor.handlers import ProcessorReportHandler from powerapi.handler import StartHandler from powerapi.report import Report @@ -43,13 +43,13 @@ libvirtError = LibvirtException -class LibvirtProcessorReportHandler(ReportHandler): +class LibvirtProcessorReportHandler(ProcessorReportHandler): """ Modify reports by replacing libvirt id by open stak uuid """ def __init__(self, state): - ReportHandler.__init__(self, state=state) + ProcessorReportHandler.__init__(self, state=state) def handle(self, report: Report): """ diff --git a/powerapi/processor/processor_actor.py b/powerapi/processor/processor_actor.py index e46ac9a4..95e3dc15 100644 --- a/powerapi/processor/processor_actor.py +++ b/powerapi/processor/processor_actor.py @@ -57,7 +57,7 @@ class ProcessorActor(Actor): """ ProcessorActor class - A processor modifies a report and sends the modified report to a list of target + A processor modifies a report and sends the modified report to a list of targets actor. """ @@ -73,10 +73,9 @@ def __init__(self, name: str, target_actors: list = None, level_logger: int = lo def setup(self): """ - Define StartMessage handler and PoisonPillMessage handler and ReportMessage handler + Define PoisonPillMessage handler """ self.add_handler(message_type=PoisonPillMessage, handler=PoisonPillMessageHandler(state=self.state)) - # self.add_handler(message_type=StartMessage, handler=StartHandler(state=self.state)) def add_target_actor(self, actor: Actor): """ diff --git a/tests/unit/actor/abstract_test_actor.py b/tests/unit/actor/abstract_test_actor.py index b706ff0b..e34db9a3 100644 --- a/tests/unit/actor/abstract_test_actor.py +++ b/tests/unit/actor/abstract_test_actor.py @@ -206,6 +206,7 @@ def handle_message(self, msg: CrashMessage): PUSHER_NAME_POWER_REPORT = 'fake_pusher_power' PUSHER_NAME_HWPC_REPORT = 'fake_pusher_hwpc' +TARGET_ACTOR_NAME = 'fake_target_actor' REPORT_TYPE_TO_BE_SENT = PowerReport REPORT_TYPE_TO_BE_SENT_2 = HWPCReport @@ -236,6 +237,13 @@ def actor(self): """ raise NotImplementedError() + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + raise NotImplementedError() + @pytest.fixture def init_actor(self, actor): actor.start() @@ -265,6 +273,18 @@ def started_fake_pusher_power_report(self, dummy_pipe_in): join_actor(pusher) + @pytest.fixture + def started_fake_target_actor(self, report_to_be_sent, dummy_pipe_in): + """ + Return a started DummyActor. When the test is finished, the actor is stopped + """ + target_actor = DummyActor(name=TARGET_ACTOR_NAME, pipe=dummy_pipe_in, message_type=report_to_be_sent) + target_actor.start() + + yield target_actor + if target_actor.is_alive(): + target_actor.terminate() + @pytest.fixture def started_fake_pusher_hwpc_report(self, dummy_pipe_in): pusher = DummyActor(PUSHER_NAME_HWPC_REPORT, dummy_pipe_in, REPORT_TYPE_TO_BE_SENT_2) @@ -316,6 +336,7 @@ def test_send_PoisonPillMessage_set_actor_alive_to_False(self, init_actor): def test_send_StartMessage_answer_OkMessage(self, init_actor): init_actor.send_control(StartMessage(SENDER_NAME)) msg = init_actor.receive_control(2000) + print('message start', str(msg)) assert isinstance(msg, OKMessage) def test_send_StartMessage_to_already_started_actor_answer_ErrorMessage(self, started_actor): diff --git a/tests/unit/processor/conftest.py b/tests/unit/processor/conftest.py new file mode 100644 index 00000000..8c3d7444 --- /dev/null +++ b/tests/unit/processor/conftest.py @@ -0,0 +1,223 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from unittest.mock import Mock + +import pytest + +from powerapi.message import K8sPodUpdateMessage +from powerapi.processor.k8s.k8s_processor_actor import K8sMetadataCache + + +@pytest.fixture(name='pods_list') +def basic_pods_list(): + """ + Return a list of three pods + """ + return ['pod1', 'pod2', 'pod3'] + + +class FakeMetadata: + """ + Fake metadata class related to an event + """ + + def __init__(self, name: str, namespace: str, labels: list): + self.name = name + self.namespace = namespace + self.labels = labels + + +class FakeContainerStatus: + """ + Fake container status infos related to an event + """ + + def __init__(self, container_id: str): + self.container_id = container_id + + +class FakeStatus: + """ + Fake status infos related to an event + """ + + def __init__(self, container_statuses: list): + self.container_statuses = container_statuses + + +class FakePod: + """ + Fake pod class related to an event + """ + + def __init__(self, metadata: FakeMetadata, status: FakeStatus): + self.metadata = metadata + self.status = status + + +@pytest.fixture(name='events_list_k8s') +def basic_events_list_k8s(): + """ + Return a list of three events + """ + return [{ + 'type': 'ADDED', + 'object': FakePod(metadata=FakeMetadata(name='o1', namespace='s1', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[FakeContainerStatus(container_id='test://s1c1'), + FakeContainerStatus(container_id='test://s1c2'), + FakeContainerStatus(container_id='test://s1c3')])) + + }, + { + 'type': 'MODIFIED', + 'object': FakePod(metadata=FakeMetadata(name='o2', namespace='s2', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[FakeContainerStatus(container_id='test://s2c1'), + FakeContainerStatus(container_id='test://s2c2'), + FakeContainerStatus(container_id='test://s2c3')]))}, + { + 'type': 'DELETED', + 'object': FakePod(metadata=FakeMetadata(name='o3', namespace='s3', labels=['l1', 'l2', 'l3', 'l4', 'l5']), + status=FakeStatus(container_statuses=[]))} + + ] + + +@pytest.fixture(name='unknown_events_list_k8s') +def basic_unknown_events_list_k8s(events_list_k8s): + """ + Modify and return an event list with unknown events types + """ + event_count = len(events_list_k8s) + + for event_indice in range(event_count): + events_list_k8s[event_indice]['type'] = 'Unknown_' + str(event_indice) + + return events_list_k8s + + +@pytest.fixture(name='expected_events_list_k8s') +def expected_basic_events_list_k8s(events_list_k8s): + """ + Return the expected list of event information according to a list of events + """ + events = [] + + for event_infos in events_list_k8s: + events.append((event_infos['type'], event_infos['object'].metadata.namespace, + event_infos['object'].metadata.name, + extract_containers_ids(event_infos['object'].status.container_statuses), + event_infos['object'].metadata.labels)) + + return events + + +def extract_containers_ids(containers_status: list) -> list: + """ + Return the containers ids by using the given list of containers status + """ + containers_ids = [] + for container_status in containers_status: + containers_ids.append(container_status.container_id[container_status.container_id.find('//') + 2: + len(container_status.container_id)]) + + return containers_ids + + +@pytest.fixture +def expected_k8s_pod_update_messages(expected_events_list_k8s): + """ + Return a list of K8sPodUpdateMessage by using the provided events list + """ + update_messages = [] + + for type, namespace, name, containers_id, labels in expected_events_list_k8s: + update_messages.append(K8sPodUpdateMessage(sender_name='test_k8s_monitor_actor', + event=type, + namespace=namespace, + pod=name, + containers_id=containers_id, + labels=labels)) + + return update_messages + + +class MockedWatch(Mock): + """ + Mocked class for simulating the Watch class from K8s API + """ + + def __init__(self, events): + Mock.__init__(self) + self.events = events + self.args = None + + def stream(self, *args): + """ + Return the list of events related to the MockedWatch + """ + self.args = args + return self.events + + +@pytest.fixture +def mocked_watch_initialized(events_list_k8s): + """ + Return a MockedWatch with the event list + """ + return MockedWatch(events_list_k8s) + + +@pytest.fixture +def mocked_watch_initialized_unknown_events(unknown_events_list_k8s): + """ + Return a MockedWatch with a list of unknown events + """ + return MockedWatch(unknown_events_list_k8s) + + +class PipeMetadataCache(K8sMetadataCache): + """ + K8sMetadataCache maintains a cache of pods' metadata + (namespace, labels and id of associated containers). + This metadata cache send itself via a pipe when an update is done + """ + + def __init__(self, name: str, level_logger: int, pipe): + K8sMetadataCache.__init__(self, name=name, level_logger=level_logger) + self.pipe = pipe + + def update_cache(self, message: K8sPodUpdateMessage): + """ + Update the local cache for pods. + + Send the metadata cache via the pipe + """ + K8sMetadataCache.update_cache(self, message=message) + self.pipe.send(self) diff --git a/tests/unit/processor/k8s/__init__.py b/tests/unit/processor/k8s/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/k8s/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/k8s/test_k8s_monitor.py b/tests/unit/processor/k8s/test_k8s_monitor.py new file mode 100644 index 00000000..150690d1 --- /dev/null +++ b/tests/unit/processor/k8s/test_k8s_monitor.py @@ -0,0 +1,121 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=R6301,W0613,W0221 + +from unittest.mock import patch, Mock + +import pytest + +from kubernetes import client + +from powerapi.message import K8sPodUpdateMessage +from powerapi.processor.k8s.k8s_monitor_actor import local_config, MANUAL_CONFIG_MODE, \ + K8sMonitorAgentActor +from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe + +LISTENER_AGENT_NAME = 'test_k8s_processor_listener_agent' + + +def test_load_local_config(): + """ + Test that load_config works correctly + """ + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock( + return_value={'pod': 'some infos about the pod...'}))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + local_config() + + # Just check we are able to make a request and get a non-empty response + v1_api = client.CoreV1Api() + ret = v1_api.list_pod_for_all_namespaces() + assert ret.items != [] + + +class TestK8sMonitor(AbstractTestActor): + """ + Class for testing a monitor actor + """ + + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + return K8sPodUpdateMessage + + @pytest.fixture + def actor(self, started_fake_target_actor, mocked_watch_initialized, pods_list): + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): + yield K8sMonitorAgentActor(name='test_k8s_monitor_actor', + listener_agent=started_fake_target_actor, + k8s_api_mode=MANUAL_CONFIG_MODE) + + def test_streaming_query(self, started_actor, pods_list, expected_events_list_k8s, mocked_watch_initialized, + shutdown_system): + """ + Test that k8s_streaming_query is able to retrieve events related to pods + """ + result = started_actor.k8s_streaming_query(timeout_seconds=5, k8sapi_mode=MANUAL_CONFIG_MODE) + + assert result == expected_events_list_k8s + + def test_unknown_events_streaming_query(self, pods_list, mocked_watch_initialized_unknown_events, + started_actor, shutdown_system): + """ + Test that unknown events are ignored by k8s_streaming_query + """ + result = started_actor.k8s_streaming_query(timeout_seconds=5, k8sapi_mode=MANUAL_CONFIG_MODE) + + assert result == [] + + def test_monitor_send_message_k8s_pod_update_message_when_events_are_available(self, started_actor, + expected_k8s_pod_update_messages, + dummy_pipe_out, shutdown_system): + """ + Test that the monitor sends to the target an update message when events are available + """ + messages_found = 0 + + for _ in range(len(expected_k8s_pod_update_messages)): + result = recv_from_pipe(dummy_pipe_out, 2) + got_message = result[1] + assert isinstance(got_message, K8sPodUpdateMessage) + + for expected_message in expected_k8s_pod_update_messages: + + if got_message == expected_message: + messages_found += 1 + break + + assert messages_found == len(expected_k8s_pod_update_messages) diff --git a/tests/unit/processor/k8s/test_k8s_processor.py b/tests/unit/processor/k8s/test_k8s_processor.py new file mode 100644 index 00000000..fd429849 --- /dev/null +++ b/tests/unit/processor/k8s/test_k8s_processor.py @@ -0,0 +1,298 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# pylint: disable=R6301,W0221,W0613 + +import logging +from copy import deepcopy +from unittest.mock import patch, Mock + +import pytest + +from powerapi.message import K8sPodUpdateMessage +from powerapi.processor.k8s.k8s_monitor_actor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT +from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor +from powerapi.processor.k8s.k8s_processor_handlers import clean_up_container_id, POD_NAMESPACE_METADATA_KEY, \ + POD_NAME_METADATA_KEY +from powerapi.report import HWPCReport +from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe +from tests.unit.processor.conftest import PipeMetadataCache +from tests.utils.report.hwpc import extract_rapl_reports_with_2_sockets + +DISPATCHER_NAME = 'test_k8s_processor_dispatcher' + + +def test_clean_up_id_docker(): + """ + Test that the cleanup of the docker id works correctly + """ + r = clean_up_container_id( + "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod435532e3_546d_45e2_8862_d3c7b320d2d9.slice/" + "docker-68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a.scope") + + assert r == "68aa4b590997e0e81257ac4a4543d5b278d70b4c279b4615605bb48812c9944a" + + +def test_clean_up_id_othercri(): + """ + Test that the cleanup of the docker id works correctly + """ + r = clean_up_container_id( + "/kubepods/besteffort/pod42006d2c-cad7-4575-bfa3-91848a558743/ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5") + + assert r == "ba28184d18d3fc143d5878c7adbefd7d1651db70ca2787f40385907d3304e7f5" + + +class TestK8sProcessor(AbstractTestActor): + """ + Class for testing a K8s Processor Actor + """ + + @pytest.fixture + def multiprocess_metadata_cache_empty(self, dummy_pipe_in): + """ + Create a metadata cache that send the object once it is modified via a pipe + """ + return PipeMetadataCache(name='test_k8s_process', level_logger=logging.DEBUG, pipe=dummy_pipe_in) + + @pytest.fixture + def update_metadata_cache_message_added_event(self): + """ + Create a update message + """ + return K8sPodUpdateMessage(sender_name='test_k8s_processor_added', + event=ADDED_EVENT, + namespace='test_k8s_processor_namespace', + pod='test_k8s_processor_pod', + containers_id=[ + 'test_cid_1_added', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_2_added', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_3_added', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_4_added'], + labels={'l1': 'v1', 'l2': 'v2', 'l3': 'v3'}) + + @pytest.fixture + def update_metadata_cache_message_modified_event(self): + """ + Create a update message + """ + return K8sPodUpdateMessage(sender_name='test_k8s_processor_modified', + event=MODIFIED_EVENT, + namespace='test_k8s_processor_namespace', + pod='test_k8s_processor_pod', + containers_id=[ + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_1_modified', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_2_modified', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_3_modified', + '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_4_modified'], + labels={'l1': 'v1_modified', 'l2_modified': 'v2_modified', + 'l3_modified': 'v3_modified'}) + + @pytest.fixture + def update_metadata_cache_message_deleted_event(self): + """ + Create a update message with DELETED as event + """ + return K8sPodUpdateMessage(sender_name='test_k8s_processor_deleted', + event=DELETED_EVENT, + namespace='test_k8s_processor_namespace', + pod='test_k8s_processor_pod') + + @pytest.fixture + def update_metadata_cache_message_unknown_event(self): + """ + Create a update message + """ + return K8sPodUpdateMessage(sender_name='test_k8s_processor_unknown', + event='Unknown Event', + namespace='test_k8s_processor_namespace', + pod='test_k8s_processor_pod') + + @pytest.fixture + def init_multiprocess_metadata_cache_with_data(self, started_actor, update_metadata_cache_message_added_event, + dummy_pipe_out): + """ + Initialize the metadata cache of the actor + """ + started_actor.send_data(update_metadata_cache_message_added_event) + _ = recv_from_pipe(dummy_pipe_out, 2) + return started_actor + + @pytest.fixture() + def hwpc_report(self, update_metadata_cache_message_added_event): + """ + Return a HWPC Report + """ + json_input = extract_rapl_reports_with_2_sockets(1)[0] + report = HWPCReport.from_json(json_input) + report.target = update_metadata_cache_message_added_event.containers_id[0] + + return report + + @pytest.fixture() + def hwpc_report_with_metadata(self, hwpc_report, update_metadata_cache_message_added_event): + """ + Return a HWPC report with metadata + """ + hwpc_report_with_metadata = deepcopy(hwpc_report) + + hwpc_report_with_metadata.metadata[POD_NAMESPACE_METADATA_KEY] = \ + update_metadata_cache_message_added_event.namespace + hwpc_report_with_metadata.metadata[POD_NAME_METADATA_KEY] = update_metadata_cache_message_added_event.pod + + for label_name, label_value in update_metadata_cache_message_added_event.labels.items(): + hwpc_report_with_metadata.metadata[f"label_{label_name}"] = label_value + + return hwpc_report_with_metadata + + @pytest.fixture + def report_to_be_sent(self): + """ + This fixture must return the report class for testing + """ + return HWPCReport + + @pytest.fixture + def actor(self, started_fake_target_actor, pods_list, mocked_watch_initialized, + multiprocess_metadata_cache_empty): + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): + with patch('powerapi.processor.k8s.k8s_processor_actor.K8sMetadataCache', + return_value=multiprocess_metadata_cache_empty): + return K8sProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, + target_actors=[started_fake_target_actor], level_logger=logging.DEBUG) + + def test_update_metadata_cache_with_added_event(self, started_actor, update_metadata_cache_message_added_event, + dummy_pipe_out, shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of an added event + """ + update_message = update_metadata_cache_message_added_event + started_actor.send_data(update_message) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message.labels + assert result.pod_containers[(update_message.namespace, update_message.pod)] == update_message.containers_id + + for container_id in update_message.containers_id: + assert result.containers_pod[container_id] == (update_message.namespace, update_message.pod) + + def test_update_metadata_cache_with_modified_event(self, init_multiprocess_metadata_cache_with_data, + update_metadata_cache_message_modified_event, dummy_pipe_out, + shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of a modified event + """ + started_actor = init_multiprocess_metadata_cache_with_data + + update_message = update_metadata_cache_message_modified_event + started_actor.send_data(update_message) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message.labels + assert result.pod_containers[(update_message.namespace, update_message.pod)] == update_message.containers_id + + for container_id in update_message.containers_id: + assert result.containers_pod[container_id] == (update_message.namespace, update_message.pod) + + def test_update_metadata_cache_with_deleted_event(self, init_multiprocess_metadata_cache_with_data, + update_metadata_cache_message_deleted_event, dummy_pipe_out, + shutdown_system): + """ + Test that metadata_cache is correctly updated when a reception of a deleted event + """ + started_actor = init_multiprocess_metadata_cache_with_data + + update_message = update_metadata_cache_message_deleted_event + + started_actor.send_data(update_message) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert len(result.pod_labels) == 0 + assert len(result.pod_containers) == 0 + assert len(result.containers_pod) == 0 + + def test_update_metadata_cache_with_unknown_event_does_not_modify_it(self, + init_multiprocess_metadata_cache_with_data, + update_metadata_cache_message_unknown_event, + dummy_pipe_out, + update_metadata_cache_message_added_event, + shutdown_system): + """ + Test that metadata_cache is not updated when a reception of a unknown event + """ + started_actor = init_multiprocess_metadata_cache_with_data + + update_message = update_metadata_cache_message_unknown_event + update_message_added = update_metadata_cache_message_added_event + started_actor.send_data(update_message) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message_added.labels + assert result.pod_containers[(update_message.namespace, update_message.pod)] == \ + update_message_added.containers_id + + for container_id in update_message.containers_id: + assert result.containers_pod[container_id] == (update_message_added.namespace, update_message_added.pod) + + def test_add_metadata_to_hwpc_report(self, + init_multiprocess_metadata_cache_with_data, + hwpc_report, hwpc_report_with_metadata, + dummy_pipe_out, shutdown_system): + """ + Test that a HWPC report is modified with the correct metadata + """ + started_actor = init_multiprocess_metadata_cache_with_data + + started_actor.send_data(hwpc_report) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result[1] == hwpc_report_with_metadata + + def test_add_metadata_to_hwpc_report_does_not_modifie_report_with_unknown_container_id(self, + started_actor, + hwpc_report, + dummy_pipe_out, + shutdown_system): + """ + Test that a HWPC report is not modified with an unknown container id + """ + + started_actor.send_data(hwpc_report) + + result = recv_from_pipe(dummy_pipe_out, 2) + + assert result[1] == hwpc_report diff --git a/tests/utils/actor/dummy_handlers.py b/tests/utils/actor/dummy_handlers.py index 8f03040c..de9b773c 100644 --- a/tests/utils/actor/dummy_handlers.py +++ b/tests/utils/actor/dummy_handlers.py @@ -48,4 +48,4 @@ def handle(self, msg: Message): :param Object msg: the message received by the actor """ self.state.pipe.send((self.state.actor.name, msg)) - logging.debug('receive : ' + str(msg), extra={'actor_name': self.state.actor.name}) + logging.debug('received : ' + str(msg), extra={'actor_name': self.state.actor.name}) From dc192b92edfd18400ee40a32d189136a74db4445 Mon Sep 17 00:00:00 2001 From: daniel Date: Fri, 18 Aug 2023 10:43:57 +0200 Subject: [PATCH 05/35] test: Add some test on generator regarding K8s Processors --- powerapi/cli/generator.py | 2 +- tests/unit/cli/conftest.py | 25 ++++++++ tests/unit/cli/test_generator.py | 63 +++++++++++++++++++ .../cli/k8s_processor_configuration.json | 10 +++ .../several_k8s_processors_configuration.json | 40 ++++++++++++ ..._without_some_arguments_configuration.json | 22 +++++++ 6 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 tests/utils/cli/k8s_processor_configuration.json create mode 100644 tests/utils/cli/several_k8s_processors_configuration.json create mode 100644 tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index c807160b..eab2e1b2 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -63,7 +63,7 @@ REGEXP_KEY = 'regexp' K8S_API_MODE_KEY = 'ks8_api_mode' TIME_INTERVAL_KEY = 'time_interval' -TIMEOUT_QUERY_KEY = 'time_interval' +TIMEOUT_QUERY_KEY = 'timeout_query' GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 84419f3e..f52bc8da 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -203,6 +203,23 @@ def several_libvirt_processors_without_some_arguments_config(): file_name='several_libvirt_processors_without_some_arguments_configuration.json') +@pytest.fixture +def several_k8s_processors_config(): + """ + Configuration with several k8s processors + """ + return load_configuration_from_json_file(file_name='several_k8s_processors_configuration.json') + + +@pytest.fixture +def several_k8s_processors_without_some_arguments_config(): + """ + Configuration with several k8s processors + """ + return load_configuration_from_json_file( + file_name='several_k8s_processors_without_some_arguments_configuration.json') + + @pytest.fixture def csv_io_postmortem_config(invalid_csv_io_stream_config): """ @@ -260,6 +277,14 @@ def libvirt_processor_config(): return load_configuration_from_json_file(file_name='libvirt_processor_configuration.json') +@pytest.fixture +def k8s_processor_config(): + """ + Configuration with k8s as processor + """ + return load_configuration_from_json_file(file_name='k8s_processor_configuration.json') + + @pytest.fixture() def subgroup_parser(): """ diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 394203d0..820814db 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -35,6 +35,8 @@ from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist +from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ + TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor @@ -455,3 +457,64 @@ def test_generate_libvirt_processor_raise_exception_when_missing_arguments( with pytest.raises(PowerAPIException): generator.generate(several_libvirt_processors_without_some_arguments_config) + + +def test_generate_processor_from_k8s_config(k8s_processor_config): + """ + Test that generation for k8s processor from a config works correctly + """ + generator = ProcessorGenerator() + + processors = generator.generate(k8s_processor_config) + + assert len(processors) == len(k8s_processor_config) + assert 'my_processor' in processors + processor = processors['my_processor'] + + assert isinstance(processor, K8sProcessorActor) + + assert processor.state.monitor_agent is None + assert processor.state.k8s_api_mode == k8s_processor_config["processor"]["my_processor"]["ks8_api_mode"] + assert processor.state.time_interval == k8s_processor_config["processor"]["my_processor"]["time_interval"] + assert processor.state.timeout_query == k8s_processor_config["processor"]["my_processor"]["timeout_query"] + + +def test_generate_several_k8s_processors_from_config(several_k8s_processors_config): + """ + Test that several k8s processors are correctly generated + """ + generator = ProcessorGenerator() + + processors = generator.generate(several_k8s_processors_config) + + assert len(processors) == len(several_k8s_processors_config['processor']) + + for processor_name, current_processor_infos in several_k8s_processors_config['processor'].items(): + assert processor_name in processors + assert isinstance(processors[processor_name], K8sProcessorActor) + + assert processors[processor_name].state.monitor_agent is None + assert processors[processor_name].state.k8s_api_mode == current_processor_infos["ks8_api_mode"] + assert processors[processor_name].state.time_interval == current_processor_infos["time_interval"] + assert processors[processor_name].state.timeout_query == current_processor_infos["timeout_query"] + + +def test_generate_k8s_processor_uses_default_values_with_missing_arguments( + several_k8s_processors_without_some_arguments_config): + """ + Test that ProcessorGenerator generates a processor with default values when arguments are not defined + """ + generator = ProcessorGenerator() + + processors = generator.generate(several_k8s_processors_without_some_arguments_config) + + assert len(processors) == len(several_k8s_processors_without_some_arguments_config['processor']) + + for processor_name, current_processor_infos in several_k8s_processors_without_some_arguments_config['processor'].items(): + assert processor_name in processors + assert isinstance(processors[processor_name], K8sProcessorActor) + + assert processors[processor_name].state.monitor_agent is None + assert processors[processor_name].state.k8s_api_mode is None + assert processors[processor_name].state.time_interval == TIME_INTERVAL_DEFAULT_VALUE + assert processors[processor_name].state.timeout_query == TIMEOUT_QUERY_DEFAULT_VALUE diff --git a/tests/utils/cli/k8s_processor_configuration.json b/tests/utils/cli/k8s_processor_configuration.json new file mode 100644 index 00000000..b6fae05a --- /dev/null +++ b/tests/utils/cli/k8s_processor_configuration.json @@ -0,0 +1,10 @@ +{ + "processor": { + "my_processor": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 20, + "timeout_query": 30 + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_processors_configuration.json b/tests/utils/cli/several_k8s_processors_configuration.json new file mode 100644 index 00000000..384fea6b --- /dev/null +++ b/tests/utils/cli/several_k8s_processors_configuration.json @@ -0,0 +1,40 @@ +{ + "processor": { + "my_processor_1": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 20, + "timeout_query": 20 + }, + "my_processor_2": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 30, + "timeout_query": 30 + }, + "my_processor_3": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 40, + "timeout_query": 40 + }, + "my_processor_4": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 50 + }, + "my_processor_5": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 60, + "timeout_query": 60 + }, + "my_processor_6": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 70, + "timeout_query": 70 + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json new file mode 100644 index 00000000..a2729438 --- /dev/null +++ b/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json @@ -0,0 +1,22 @@ +{ + "processor": { + "my_processor_1": { + "type": "k8s" + }, + "my_processor_2": { + "type": "k8s" + }, + "my_processor_3": { + "type": "k8s" + }, + "my_processor_4": { + "type": "k8s" + }, + "my_processor_5": { + "type": "k8s" + }, + "my_processor_6": { + "type": "k8s" + } + } +} \ No newline at end of file From 075ddd90e1c0cd706b49448873e6eba80f7c433a Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 21 Aug 2023 15:19:21 +0200 Subject: [PATCH 06/35] test: Add test for binding regarding K8s Processors, add Monitor Generator and related tests --- powerapi/cli/binding_manager.py | 2 +- powerapi/cli/generator.py | 38 ++++++- powerapi/exception.py | 12 ++ powerapi/processor/k8s/k8s_monitor_actor.py | 2 +- powerapi/processor/k8s/k8s_processor_actor.py | 8 +- tests/unit/cli/conftest.py | 85 +++++++++++---- tests/unit/cli/test_binding_manager.py | 42 +++---- tests/unit/cli/test_generator.py | 103 ++++++++++++++---- ...ith_non_existent_puller_configuration.json | 36 ++++++ ...o_k8s_processor_binding_configuration.json | 36 ++++++ ...processor_wrong_binding_configuration.json | 36 ++++++ 11 files changed, 331 insertions(+), 69 deletions(-) create mode 100644 tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json create mode 100644 tests/utils/cli/puller_to_k8s_processor_binding_configuration.json create mode 100644 tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py index 80dbb827..4c955cfb 100644 --- a/powerapi/cli/binding_manager.py +++ b/powerapi/cli/binding_manager.py @@ -108,7 +108,7 @@ def process_bindings(self, bindings: dict): the "" and "to": "" follow the convention "." according to the configuration, e.g., "input.my_puller" and "processor.my_libvirt_processor" - One of the actors in the binding hs to be a processor. If the "to" actor is the processor, the "from" has to be + One of the actors in the binding has to be a processor. If the "to" actor is the processor, the "from" has to be a puller. If the "from" actor is a processor, the "to" actor has to be a pusher. :param bindings: The bindings to be processed. """ diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index eab2e1b2..3905299e 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -35,8 +35,9 @@ from powerapi.actor import Actor from powerapi.database.influxdb2 import InfluxDB2 from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ - DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed + DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed, MonitorTypeDoesNotExist from powerapi.filter import Filter +from powerapi.processor.k8s.k8s_monitor_actor import K8sMonitorAgentActor from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor @@ -65,9 +66,13 @@ TIME_INTERVAL_KEY = 'time_interval' TIMEOUT_QUERY_KEY = 'timeout_query' +LISTENER_ACTOR_KEY = 'listener_actor' + GENERAL_CONF_STREAM_MODE_KEY = 'stream' GENERAL_CONF_VERBOSE_KEY = 'verbose' +MONITOR_NAME_SUFFIX = '_monitor' + class Generator: """ @@ -410,3 +415,34 @@ def _actor_factory(self, actor_name: str, _, component_config: dict): else: component_config[ACTOR_NAME_KEY] = actor_name return self.processor_factory[processor_actor_type](component_config) + + +class MonitorGenerator(Generator): + """ + Generator that initialises the monitor by using a K8sProcessorActor + """ + + def __init__(self): + Generator.__init__(self, component_group_name='monitor') + + self.monitor_factory = { + 'k8s': lambda monitor_config: K8sMonitorAgentActor( + name=monitor_config[ACTOR_NAME_KEY], + listener_agent=monitor_config[LISTENER_ACTOR_KEY], + k8s_api_mode=monitor_config[LISTENER_ACTOR_KEY].state.k8s_api_mode, + time_interval=monitor_config[LISTENER_ACTOR_KEY].state.time_interval, + timeout_query=monitor_config[LISTENER_ACTOR_KEY].state.timeout_query, + level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel() + ) + + } + + def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str): + + monitor_actor_type = component_config[COMPONENT_TYPE_KEY] + + if monitor_actor_type not in self.monitor_factory: + raise MonitorTypeDoesNotExist(monitor_type=monitor_actor_type) + else: + component_config[ACTOR_NAME_KEY] = actor_name + MONITOR_NAME_SUFFIX + return self.monitor_factory[monitor_actor_type](component_config) diff --git a/powerapi/exception.py b/powerapi/exception.py index de52bb8f..968d5c2a 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -285,6 +285,7 @@ class LibvirtException(PowerAPIException): """ Exception raised when there are issues regarding the import of LibvirtException """ + def __init__(self, _): PowerAPIException.__init__(self) @@ -325,3 +326,14 @@ class UnknownMessageTypeException(PowerAPIException): """ Exception happen when we don't know the message type """ + + +class MonitorTypeDoesNotExist(PowerAPIException): + """ + Exception raised when attempting to remove to a MonitorGenerator a monitor factory with a type that is not + bound to a monitor factory + """ + + def __init__(self, monitor_type: str): + PowerAPIException.__init__(self) + self.monitor_type = monitor_type diff --git a/powerapi/processor/k8s/k8s_monitor_actor.py b/powerapi/processor/k8s/k8s_monitor_actor.py index 4b16db31..89962b09 100644 --- a/powerapi/processor/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/k8s/k8s_monitor_actor.py @@ -162,7 +162,7 @@ def __init__(self, name: str, listener_agent: Actor, k8s_api_mode: str = None, t :param int time_interval: Time interval for the monitoring :pram int level_logger: The logger level """ - Actor.__init__(self, name=name, level_logger=logging.DEBUG) + Actor.__init__(self, name=name, level_logger=level_logger) self.state = K8sMonitorAgentState(actor=self, time_interval=time_interval, timeout_query=timeout_query, listener_agent=listener_agent, k8s_api_mode=k8s_api_mode) diff --git a/powerapi/processor/k8s/k8s_processor_actor.py b/powerapi/processor/k8s/k8s_processor_actor.py index 70f7e639..83d9e787 100644 --- a/powerapi/processor/k8s/k8s_processor_actor.py +++ b/powerapi/processor/k8s/k8s_processor_actor.py @@ -135,12 +135,10 @@ class K8sProcessorState(ProcessorState): State related to a K8SProcessorActor """ - def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, - monitor_agent_name: str, k8s_api_mode: str, time_interval: int, timeout_query: int): + def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, k8s_api_mode: str, + time_interval: int, timeout_query: int): ProcessorState.__init__(self, actor=actor, target_actors=target_actors) self.metadata_cache = metadata_cache - self.monitor_agent = None - self.monitor_agent_name = monitor_agent_name self.k8s_api_mode = k8s_api_mode self.time_interval = time_interval self.timeout_query = timeout_query @@ -158,7 +156,7 @@ def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, lev timeout=timeout) self.state = K8sProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), - monitor_agent_name=DEFAULT_K8S_MONITOR_NAME, target_actors=target_actors, + target_actors=target_actors, k8s_api_mode=ks8_api_mode, time_interval=time_interval, timeout_query=timeout_query) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index f52bc8da..b58aaac5 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -32,7 +32,8 @@ import pytest import tests.utils.cli as test_files_module from powerapi.cli.binding_manager import INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, ProcessorBindingManager -from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator +from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator, COMPONENT_TYPE_KEY, \ + LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX from powerapi.dispatcher import DispatcherActor, RouteTable from powerapi.filter import Filter from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ @@ -220,6 +221,31 @@ def several_k8s_processors_without_some_arguments_config(): file_name='several_k8s_processors_without_some_arguments_configuration.json') +def generate_k8s_monitors_config(processors_config): + """ + Generate the configuration related to the given k8s processors configuration + """ + generator = ProcessorGenerator() + + processors = generator.generate(processors_config) + + monitors = {'monitor': {}} + + for processor_name, processor in processors.items(): + monitors['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', + LISTENER_ACTOR_KEY: processor} + + return monitors + + +@pytest.fixture +def several_k8s_monitors_config(several_k8s_processors_config): + """ + Configuration with several k8s monitors derived from processor generators + """ + return generate_k8s_monitors_config(several_k8s_processors_config) + + @pytest.fixture def csv_io_postmortem_config(invalid_csv_io_stream_config): """ @@ -285,6 +311,24 @@ def k8s_processor_config(): return load_configuration_from_json_file(file_name='k8s_processor_configuration.json') +@pytest.fixture +def k8s_monitor_config(k8s_processor_config): + """ + the configuration of k8s monitors is derived from processor generators + """ + generator = ProcessorGenerator() + + processors = generator.generate(k8s_processor_config) + + monitors = {'monitor': {}} + + for processor_name, processor in processors.items(): + monitors['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', + LISTENER_ACTOR_KEY: processor} + + return monitors + + @pytest.fixture() def subgroup_parser(): """ @@ -529,33 +573,36 @@ def empty_cli_configuration(): sys.argv = [] -@pytest.fixture -def puller_to_libvirt_processor_binding_configuration(): +@pytest.fixture(params=['puller_to_libvirt_processor_binding_configuration.json', + 'puller_to_k8s_processor_binding_configuration.json']) +def puller_to_processor_binding_configuration(request): """ - Return a dictionary containing bindings with a libvirt processor + Return a dictionary containing bindings with a processor """ - return load_configuration_from_json_file(file_name='puller_to_libvirt_processor_binding_configuration.json') + return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture -def pusher_to_libvirt_processor_wrong_binding_configuration(): +@pytest.fixture(params=['pusher_to_libvirt_processor_wrong_binding_configuration.json', + 'pusher_to_k8s_processor_wrong_binding_configuration.json']) +def pusher_to_processor_wrong_binding_configuration(request): """ - Return a dictionary containing bindings with a libvirt processor with a bind to a pusher + Return a dictionary containing wrong bindings with a processor """ - return load_configuration_from_json_file(file_name='pusher_to_libvirt_processor_wrong_binding_configuration.json') + return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture -def non_existent_puller_to_libvirt_processor_configuration(): +@pytest.fixture(params=['libvirt_processor_binding_with_non_existent_puller_configuration.json', + 'k8s_processor_binding_with_non_existent_puller_configuration.json']) +def not_existent_puller_to_processor_configuration(request): """ - Return a dictionary containing bindings with a libvirt processor with a bind to a pusher + Return a dictionary containing bindings with a puller that doesn't exist """ return load_configuration_from_json_file( - file_name='libvirt_processor_binding_with_non_existent_puller_configuration.json') + file_name=request.param) @pytest.fixture -def libvirt_processor_binding_actors_and_dictionary(puller_to_libvirt_processor_binding_configuration): +def processor_binding_actors_and_dictionary(puller_to_processor_binding_configuration): """ Return a list of dictionary which contains actors as well as the expected unified dictionary related to the actor list @@ -565,13 +612,13 @@ def libvirt_processor_binding_actors_and_dictionary(puller_to_libvirt_processor_ report_filter = Filter() puller_generator = PullerGenerator(report_filter=report_filter) - pullers = puller_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + pullers = puller_generator.generate(main_config=puller_to_processor_binding_configuration) actors.append(pullers) expected_actors_dictionary[INPUT_GROUP].update(pullers) pusher_generator = PusherGenerator() - pushers = pusher_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + pushers = pusher_generator.generate(main_config=puller_to_processor_binding_configuration) actors.append(pushers) expected_actors_dictionary[OUTPUT_GROUP].update(pushers) @@ -584,7 +631,7 @@ def libvirt_processor_binding_actors_and_dictionary(puller_to_libvirt_processor_ report_filter.filter(lambda msg: True, dispatcher) processor_generator = ProcessorGenerator() - processors = processor_generator.generate(main_config=puller_to_libvirt_processor_binding_configuration) + processors = processor_generator.generate(main_config=puller_to_processor_binding_configuration) actors.append(processors) expected_actors_dictionary[PROCESSOR_GROUP].update(processors) @@ -607,13 +654,13 @@ def dispatcher_actor_in_dictionary(): @pytest.fixture -def libvirt_processor_binding_manager(libvirt_processor_binding_actors_and_dictionary): +def processor_binding_manager(processor_binding_actors_and_dictionary): """ Return a ProcessorBindingManager with a libvirt Processor """ actors = {} - for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + for current_actors in processor_binding_actors_and_dictionary[0]: actors.update(current_actors) return ProcessorBindingManager(actors=actors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py index 09db4ba6..9f35afed 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/cli/test_binding_manager.py @@ -46,17 +46,17 @@ def check_default_processor_binding_manager_default_actors_content(processor_man assert len(processor_manager.actors[PROCESSOR_GROUP]) == 0 -def test_create_processor_binding_manager_with_actors(libvirt_processor_binding_actors_and_dictionary): +def test_create_processor_binding_manager_with_actors(processor_binding_actors_and_dictionary): """ - Test that a ProcessorBindingManager is correctly created when a actor dictionary is provided + Test that a ProcessorBindingManager is correctly created when an actor dictionary is provided """ actors = {} - for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + for current_actors in processor_binding_actors_and_dictionary[0]: actors.update(current_actors) binding_manager = ProcessorBindingManager(actors=actors) - assert binding_manager.actors == libvirt_processor_binding_actors_and_dictionary[1] + assert binding_manager.actors == processor_binding_actors_and_dictionary[1] def test_create_processor_binding_manager_without_actors(): @@ -76,7 +76,7 @@ def test_create_processor_binding_manager_raise_exception_with_wrong_actor_type( _ = ProcessorBindingManager(actors=dispatcher_actor_in_dictionary) -def test_add_actors(libvirt_processor_binding_actors_and_dictionary): +def test_add_actors(processor_binding_actors_and_dictionary): """ Test that a dictionary is correctly generated according to a list of actors """ @@ -84,10 +84,10 @@ def test_add_actors(libvirt_processor_binding_actors_and_dictionary): check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) - for actors in libvirt_processor_binding_actors_and_dictionary[0]: + for actors in processor_binding_actors_and_dictionary[0]: binding_manager.add_actors(actors) - assert binding_manager.actors == libvirt_processor_binding_actors_and_dictionary[1] + assert binding_manager.actors == processor_binding_actors_and_dictionary[1] def test_add_actors_raise_exception_with_wrong_actor_type(dispatcher_actor_in_dictionary): @@ -104,14 +104,14 @@ def test_add_actors_raise_exception_with_wrong_actor_type(dispatcher_actor_in_di check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) -def test_process_bindings_for_libvirt_processor(puller_to_libvirt_processor_binding_configuration, - libvirt_processor_binding_actors_and_dictionary): +def test_process_bindings_for_processor(puller_to_processor_binding_configuration, + processor_binding_actors_and_dictionary): """ Test that the bindings between a puller and a processor are correctly created """ actors = {} - for current_actors in libvirt_processor_binding_actors_and_dictionary[0]: + for current_actors in processor_binding_actors_and_dictionary[0]: actors.update(current_actors) binding_manager = ProcessorBindingManager(actors=actors) @@ -119,32 +119,32 @@ def test_process_bindings_for_libvirt_processor(puller_to_libvirt_processor_bind assert len(actors['one_puller'].state.report_filter.filters) == 1 assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], DispatcherActor) - binding_manager.process_bindings(bindings=puller_to_libvirt_processor_binding_configuration[BINDING_GROUP]) + binding_manager.process_bindings(bindings=puller_to_processor_binding_configuration[BINDING_GROUP]) assert len(actors['one_puller'].state.report_filter.filters) == 1 assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], ProcessorActor) assert actors['one_puller'].state.report_filter.filters[0][1] == actors['my_processor'] -def test_process_bindings_for_libvirt_processor_raise_exception_with_wrong_binding_types( - pusher_to_libvirt_processor_wrong_binding_configuration, - libvirt_processor_binding_manager): +def test_process_bindings_for_processor_raise_exception_with_wrong_binding_types( + pusher_to_processor_wrong_binding_configuration, + processor_binding_manager): """ Test that an exception is raised with a wrong type for the from actor in a binding """ with pytest.raises(BadInputData): - libvirt_processor_binding_manager.process_bindings( - bindings=pusher_to_libvirt_processor_wrong_binding_configuration[BINDING_GROUP]) + processor_binding_manager.process_bindings( + bindings=pusher_to_processor_wrong_binding_configuration[BINDING_GROUP]) -def test_process_bindings_for_libvirt_processor_raisse_exception_with_non_existent_puller( - non_existent_puller_to_libvirt_processor_configuration, - libvirt_processor_binding_manager): +def test_process_bindings_for_processor_raisse_exception_with_non_existent_puller( + not_existent_puller_to_processor_configuration, + processor_binding_manager): """ Test that an exception is raised with a non-existent puller """ with pytest.raises(BadInputData): - libvirt_processor_binding_manager.process_bindings( - bindings=non_existent_puller_to_libvirt_processor_configuration[BINDING_GROUP]) + processor_binding_manager.process_bindings( + bindings=not_existent_puller_to_processor_configuration[BINDING_GROUP]) diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 820814db..8e3e7ee9 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -33,8 +33,10 @@ import pytest -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator +from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator, \ + MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY from powerapi.cli.generator import ModelNameDoesNotExist +from powerapi.processor.k8s.k8s_monitor_actor import K8sMonitorAgentActor from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor @@ -139,7 +141,7 @@ def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_ac assert len(pullers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['input']) - for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input'].\ + for puller_name, current_puller_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['input']. \ items(): if current_puller_infos['type'] == 'csv': @@ -381,7 +383,7 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a assert len(pushers) == len(several_inputs_outputs_stream_csv_without_some_arguments_config['output']) - for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output'].\ + for pusher_name, current_pusher_infos in several_inputs_outputs_stream_csv_without_some_arguments_config['output']. \ items(): pusher_type = current_pusher_infos['type'] if pusher_type == 'csv': @@ -459,24 +461,33 @@ def test_generate_libvirt_processor_raise_exception_when_missing_arguments( generator.generate(several_libvirt_processors_without_some_arguments_config) +def check_k8s_processor_infos(processor: K8sProcessorActor, expected_processor_info: dict): + """ + Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor + """ + assert isinstance(processor, K8sProcessorActor) + + assert processor.state.k8s_api_mode == expected_processor_info["ks8_api_mode"] + assert processor.state.time_interval == expected_processor_info["time_interval"] + assert processor.state.timeout_query == expected_processor_info["timeout_query"] + + def test_generate_processor_from_k8s_config(k8s_processor_config): """ Test that generation for k8s processor from a config works correctly """ generator = ProcessorGenerator() + processor_name = 'my_processor' processors = generator.generate(k8s_processor_config) assert len(processors) == len(k8s_processor_config) - assert 'my_processor' in processors - processor = processors['my_processor'] + assert processor_name in processors - assert isinstance(processor, K8sProcessorActor) + processor = processors[processor_name] - assert processor.state.monitor_agent is None - assert processor.state.k8s_api_mode == k8s_processor_config["processor"]["my_processor"]["ks8_api_mode"] - assert processor.state.time_interval == k8s_processor_config["processor"]["my_processor"]["time_interval"] - assert processor.state.timeout_query == k8s_processor_config["processor"]["my_processor"]["timeout_query"] + check_k8s_processor_infos(processor=processor, + expected_processor_info=k8s_processor_config["processor"][processor_name]) def test_generate_several_k8s_processors_from_config(several_k8s_processors_config): @@ -491,12 +502,10 @@ def test_generate_several_k8s_processors_from_config(several_k8s_processors_conf for processor_name, current_processor_infos in several_k8s_processors_config['processor'].items(): assert processor_name in processors - assert isinstance(processors[processor_name], K8sProcessorActor) - assert processors[processor_name].state.monitor_agent is None - assert processors[processor_name].state.k8s_api_mode == current_processor_infos["ks8_api_mode"] - assert processors[processor_name].state.time_interval == current_processor_infos["time_interval"] - assert processors[processor_name].state.timeout_query == current_processor_infos["timeout_query"] + processor = processors[processor_name] + + check_k8s_processor_infos(processor=processor, expected_processor_info=current_processor_infos) def test_generate_k8s_processor_uses_default_values_with_missing_arguments( @@ -508,13 +517,65 @@ def test_generate_k8s_processor_uses_default_values_with_missing_arguments( processors = generator.generate(several_k8s_processors_without_some_arguments_config) + expected_processor_info = {'ks8_api_mode': None, 'time_interval': TIME_INTERVAL_DEFAULT_VALUE, + 'timeout_query': TIMEOUT_QUERY_DEFAULT_VALUE} + assert len(processors) == len(several_k8s_processors_without_some_arguments_config['processor']) - for processor_name, current_processor_infos in several_k8s_processors_without_some_arguments_config['processor'].items(): + for processor_name in several_k8s_processors_without_some_arguments_config['processor']: assert processor_name in processors - assert isinstance(processors[processor_name], K8sProcessorActor) - assert processors[processor_name].state.monitor_agent is None - assert processors[processor_name].state.k8s_api_mode is None - assert processors[processor_name].state.time_interval == TIME_INTERVAL_DEFAULT_VALUE - assert processors[processor_name].state.timeout_query == TIMEOUT_QUERY_DEFAULT_VALUE + processor = processors[processor_name] + + check_k8s_processor_infos(processor=processor, expected_processor_info=expected_processor_info) + + +def check_k8s_monitor_infos(monitor: K8sMonitorAgentActor, associated_processor: K8sProcessorActor): + """ + Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor + """ + + assert isinstance(monitor, K8sMonitorAgentActor) + + assert monitor.state.k8s_api_mode == associated_processor.state.k8s_api_mode + + assert monitor.state.time_interval == associated_processor.state.time_interval + + assert monitor.state.timeout_query == associated_processor.state.timeout_query + + +def test_generate_monitor_from_k8s_config(k8s_monitor_config): + """ + Test that generation for k8s monitor from a processor config works correctly + """ + generator = MonitorGenerator() + monitor_name = 'my_processor' + MONITOR_NAME_SUFFIX + + monitors = generator.generate(k8s_monitor_config) + + assert len(monitors) == len(k8s_monitor_config) + + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, + associated_processor=k8s_monitor_config['monitor'][monitor_name][LISTENER_ACTOR_KEY]) + + +def test_generate_several_k8s_monitors_from_config(several_k8s_monitors_config): + """ + Test that several k8s monitors are correctly generated + """ + generator = MonitorGenerator() + + monitors = generator.generate(several_k8s_monitors_config) + + assert len(monitors) == len(several_k8s_monitors_config['monitor']) + + for monitor_name, current_monitor_infos in several_k8s_monitors_config['monitor'].items(): + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, associated_processor=current_monitor_infos[LISTENER_ACTOR_KEY]) diff --git a/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json new file mode 100644 index 00000000..7487fe38 --- /dev/null +++ b/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json @@ -0,0 +1,36 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 60 + } + }, + "binding": { + "b1": { + "from": "input.non_existent_puller", + "to": "processor.my_processor" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json b/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json new file mode 100644 index 00000000..0b616994 --- /dev/null +++ b/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json @@ -0,0 +1,36 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 60 + } + }, + "binding": { + "b1": { + "from": "input.one_puller", + "to": "processor.my_processor" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json b/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json new file mode 100644 index 00000000..c62b80cc --- /dev/null +++ b/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json @@ -0,0 +1,36 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "processor": { + "my_processor": { + "type": "k8s", + "ks8_api_mode": "Manual", + "time_interval": 70, + "timeout_query": 80 + } + }, + "binding": { + "b1": { + "from": "output.one_pusher", + "to": "processor.my_processor" + } + } +} \ No newline at end of file From 7f32c84841a19199b3e8599966288bcbf5303e11 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 22 Aug 2023 15:16:19 +0200 Subject: [PATCH 07/35] feat: Add parsers for bindings and processors --- powerapi/cli/common_cli_parsing_manager.py | 70 +++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index d7e9257c..d410d0e1 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -37,6 +37,8 @@ POWERAPI_ENVIRONMENT_VARIABLE_PREFIX = 'POWERAPI_' POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'OUTPUT_' POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'INPUT_' +POWERAPI_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PROCESSOR_' +POWERAPI_BINDING_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'BINDING_' POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'REPORT_MODIFIER_' @@ -67,11 +69,19 @@ def __init__(self): self.add_subgroup(name='input', prefix=POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a database input : --db_input database_name ARG1 ARG2 ... ") + help_text="specify a database input : --input database_name ARG1 ARG2 ... ") self.add_subgroup(name='output', prefix=POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a database output : --db_output database_name ARG1 ARG2 ...") + help_text="specify a database output : --output database_name ARG1 ARG2 ...") + + self.add_subgroup(name='processor', + prefix=POWERAPI_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a processor : --processor processor_name ARG1 ARG2 ...") + + self.add_subgroup(name='binding', + prefix=POWERAPI_BINDING_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a binding : --binding binding_name ARG1 ARG2 ...") # Parsers @@ -423,6 +433,62 @@ def __init__(self): subgroup_parser=subparser_influx2_output ) + subparser_libvirt_processor = SubgroupConfigParsingManager("libvirt") + subparser_libvirt_processor.add_argument( + "u", "uri", help_text="libvirt daemon uri", default_value="" + ) + subparser_libvirt_processor.add_argument( + "d", + "domain_regexp", + help_text="regexp used to extract domain from cgroup string", + ) + subparser_libvirt_processor.add_argument("n", "name", help_text="") + self.add_subgroup_parser( + subgroup_name="processor", + subgroup_parser=subparser_libvirt_processor + ) + + subparser_k8s_processor = SubgroupConfigParsingManager("k8s") + subparser_k8s_processor.add_argument( + "a", "api_mode", help_text="k8s api mode (local, manual or cluster)" + ) + subparser_k8s_processor.add_argument( + "t", + "time_interval", + help_text="time interval for the k8s monitoring", + ) + subparser_k8s_processor.add_argument( + "o", + "timeout_query", + help_text="timeout for k8s queries", + ) + subparser_k8s_processor.add_argument("n", "name", help_text="") + self.add_subgroup_parser( + subgroup_name="processor", + subgroup_parser=subparser_k8s_processor + ) + + subparser_k8s_processor_binding = SubgroupConfigParsingManager("processor") + + subparser_k8s_processor_binding.add_argument( + "f", + "from", + help_text="starting actor for the binding", + ) + + subparser_k8s_processor_binding.add_argument( + "t", + "to", + help_text="end actor for the binding", + ) + + subparser_k8s_processor_binding.add_argument("n", "name", help_text="") + + self.add_subgroup_parser( + subgroup_name="binding", + subgroup_parser=subparser_k8s_processor_binding + ) + def parse_argv(self): """ """ try: From 01aca03d5c1e54a651576019c9195e23f57c3d8f Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 22 Aug 2023 15:20:02 +0200 Subject: [PATCH 08/35] feat: Add monitors generation from a dictionary of processors --- powerapi/cli/generator.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index 3905299e..8c0f50c2 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -62,7 +62,7 @@ ACTOR_NAME_KEY = 'actor_name' TARGET_ACTORS_KEY = 'target_actors' REGEXP_KEY = 'regexp' -K8S_API_MODE_KEY = 'ks8_api_mode' +K8S_API_MODE_KEY = 'k8s_api_mode' TIME_INTERVAL_KEY = 'time_interval' TIMEOUT_QUERY_KEY = 'timeout_query' @@ -72,6 +72,8 @@ GENERAL_CONF_VERBOSE_KEY = 'verbose' MONITOR_NAME_SUFFIX = '_monitor' +MONITOR_KEY = 'monitor' +K8S_COMPONENT_TYPE_VALUE = 'k8s' class Generator: @@ -423,10 +425,10 @@ class MonitorGenerator(Generator): """ def __init__(self): - Generator.__init__(self, component_group_name='monitor') + Generator.__init__(self, component_group_name=MONITOR_KEY) self.monitor_factory = { - 'k8s': lambda monitor_config: K8sMonitorAgentActor( + K8S_COMPONENT_TYPE_VALUE: lambda monitor_config: K8sMonitorAgentActor( name=monitor_config[ACTOR_NAME_KEY], listener_agent=monitor_config[LISTENER_ACTOR_KEY], k8s_api_mode=monitor_config[LISTENER_ACTOR_KEY].state.k8s_api_mode, @@ -446,3 +448,18 @@ def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str) else: component_config[ACTOR_NAME_KEY] = actor_name + MONITOR_NAME_SUFFIX return self.monitor_factory[monitor_actor_type](component_config) + + def generate_from_processors(self, processors: dict) -> dict: + """ + Generates monitors associated with the given processors + :param dict processors: Dictionary with the processors for the generation + """ + + monitors_config = {MONITOR_KEY: {}} + + for processor_name, processor in processors.items(): + monitors_config[MONITOR_KEY][processor_name + MONITOR_NAME_SUFFIX] = { + COMPONENT_TYPE_KEY: K8S_COMPONENT_TYPE_VALUE, + LISTENER_ACTOR_KEY: processor} + + return self.generate(main_config=monitors_config) From b873a43f24b0919a95f182d1a9fd035610dfd30b Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 22 Aug 2023 15:21:42 +0200 Subject: [PATCH 09/35] test: Add test for monitors generation from a processors dictionary --- tests/unit/cli/conftest.py | 37 +++++++++++++-- tests/unit/cli/test_generator.py | 45 +++++++++++++++++-- ...ith_non_existent_puller_configuration.json | 3 +- .../cli/k8s_processor_configuration.json | 2 +- ...ith_non_existent_puller_configuration.json | 1 + ...o_k8s_processor_binding_configuration.json | 5 ++- ...bvirt_processor_binding_configuration.json | 3 +- ...processor_wrong_binding_configuration.json | 3 +- ...processor_wrong_binding_configuration.json | 1 + .../several_k8s_processors_configuration.json | 12 ++--- 10 files changed, 93 insertions(+), 19 deletions(-) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index b58aaac5..3ee6502b 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -239,11 +239,29 @@ def generate_k8s_monitors_config(processors_config): @pytest.fixture -def several_k8s_monitors_config(several_k8s_processors_config): +def several_k8s_processors(several_k8s_processors_config): + """ + Return a dictionary with several k8s processors + """ + generator = ProcessorGenerator() + + processors = generator.generate(several_k8s_processors_config) + + return processors + + +@pytest.fixture +def several_k8s_monitors_config(several_k8s_processors): """ Configuration with several k8s monitors derived from processor generators """ - return generate_k8s_monitors_config(several_k8s_processors_config) + monitors_config = {'monitor': {}} + + for processor_name, processor in several_k8s_processors.items(): + monitors_config['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', + LISTENER_ACTOR_KEY: processor} + + return monitors_config @pytest.fixture @@ -312,14 +330,25 @@ def k8s_processor_config(): @pytest.fixture -def k8s_monitor_config(k8s_processor_config): +def k8s_processors(k8s_processor_config): """ - the configuration of k8s monitors is derived from processor generators + Return a dictionary of k8s processors """ generator = ProcessorGenerator() processors = generator.generate(k8s_processor_config) + return processors + + +@pytest.fixture +def k8s_monitor_config(k8s_processors): + """ + The configuration of k8s monitors is derived from processor generators + """ + + processors = k8s_processors + monitors = {'monitor': {}} for processor_name, processor in processors.items(): diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 8e3e7ee9..198026db 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -467,7 +467,7 @@ def check_k8s_processor_infos(processor: K8sProcessorActor, expected_processor_i """ assert isinstance(processor, K8sProcessorActor) - assert processor.state.k8s_api_mode == expected_processor_info["ks8_api_mode"] + assert processor.state.k8s_api_mode == expected_processor_info["k8s_api_mode"] assert processor.state.time_interval == expected_processor_info["time_interval"] assert processor.state.timeout_query == expected_processor_info["timeout_query"] @@ -517,7 +517,7 @@ def test_generate_k8s_processor_uses_default_values_with_missing_arguments( processors = generator.generate(several_k8s_processors_without_some_arguments_config) - expected_processor_info = {'ks8_api_mode': None, 'time_interval': TIME_INTERVAL_DEFAULT_VALUE, + expected_processor_info = {'k8s_api_mode': None, 'time_interval': TIME_INTERVAL_DEFAULT_VALUE, 'timeout_query': TIMEOUT_QUERY_DEFAULT_VALUE} assert len(processors) == len(several_k8s_processors_without_some_arguments_config['processor']) @@ -544,7 +544,7 @@ def check_k8s_monitor_infos(monitor: K8sMonitorAgentActor, associated_processor: assert monitor.state.timeout_query == associated_processor.state.timeout_query -def test_generate_monitor_from_k8s_config(k8s_monitor_config): +def test_generate_k8s_monitor_from_k8s_config(k8s_monitor_config): """ Test that generation for k8s monitor from a processor config works correctly """ @@ -579,3 +579,42 @@ def test_generate_several_k8s_monitors_from_config(several_k8s_monitors_config): monitor = monitors[monitor_name] check_k8s_monitor_infos(monitor=monitor, associated_processor=current_monitor_infos[LISTENER_ACTOR_KEY]) + + +def test_generate_k8s_monitor_from_k8s_processors(k8s_processors): + """ + Test that generation for k8s monitor from a processor config works correctly + """ + generator = MonitorGenerator() + processor_name = 'my_processor' + monitor_name = processor_name + MONITOR_NAME_SUFFIX + + monitors = generator.generate_from_processors(processors=k8s_processors) + + assert len(monitors) == len(k8s_processors) + + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, + associated_processor=k8s_processors[processor_name]) + + +def test_generate_several_k8s_monitors_from_processors(several_k8s_processors): + """ + Test that several k8s monitors are correctly generated + """ + generator = MonitorGenerator() + + monitors = generator.generate_from_processors(processors=several_k8s_processors) + + assert len(monitors) == len(several_k8s_processors) + + for processor_name, processor in several_k8s_processors.items(): + monitor_name = processor_name + MONITOR_NAME_SUFFIX + assert monitor_name in monitors + + monitor = monitors[monitor_name] + + check_k8s_monitor_infos(monitor=monitor, associated_processor=processor) diff --git a/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json index 7487fe38..fd907375 100644 --- a/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json +++ b/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json @@ -22,13 +22,14 @@ "processor": { "my_processor": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 50, "timeout_query": 60 } }, "binding": { "b1": { + "type": "processor", "from": "input.non_existent_puller", "to": "processor.my_processor" } diff --git a/tests/utils/cli/k8s_processor_configuration.json b/tests/utils/cli/k8s_processor_configuration.json index b6fae05a..29a14636 100644 --- a/tests/utils/cli/k8s_processor_configuration.json +++ b/tests/utils/cli/k8s_processor_configuration.json @@ -2,7 +2,7 @@ "processor": { "my_processor": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 20, "timeout_query": 30 } diff --git a/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json index 77a6951d..0a425d37 100644 --- a/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json +++ b/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json @@ -28,6 +28,7 @@ }, "binding": { "b1": { + "type": "processor", "from": "input.non_existent_puller", "to": "processor.my_processor" } diff --git a/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json b/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json index 0b616994..c3df9832 100644 --- a/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json +++ b/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json @@ -22,14 +22,15 @@ "processor": { "my_processor": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 50, "timeout_query": 60 } }, "binding": { "b1": { - "from": "input.one_puller", + "type": "processor", + "from": "input.one_puller", "to": "processor.my_processor" } } diff --git a/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json b/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json index 8aceb3bc..a6014e33 100644 --- a/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json +++ b/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json @@ -28,7 +28,8 @@ }, "binding": { "b1": { - "from": "input.one_puller", + "type": "processor", + "from": "input.one_puller", "to": "processor.my_processor" } } diff --git a/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json b/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json index c62b80cc..d8b2ece6 100644 --- a/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json +++ b/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json @@ -22,13 +22,14 @@ "processor": { "my_processor": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 70, "timeout_query": 80 } }, "binding": { "b1": { + "type": "processor", "from": "output.one_pusher", "to": "processor.my_processor" } diff --git a/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json b/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json index 3145423c..940c9e2c 100644 --- a/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json +++ b/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json @@ -28,6 +28,7 @@ }, "binding": { "b1": { + "type": "processor", "from": "output.one_pusher", "to": "processor.my_processor" } diff --git a/tests/utils/cli/several_k8s_processors_configuration.json b/tests/utils/cli/several_k8s_processors_configuration.json index 384fea6b..d56c2eeb 100644 --- a/tests/utils/cli/several_k8s_processors_configuration.json +++ b/tests/utils/cli/several_k8s_processors_configuration.json @@ -2,37 +2,37 @@ "processor": { "my_processor_1": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 20, "timeout_query": 20 }, "my_processor_2": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 30, "timeout_query": 30 }, "my_processor_3": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 40, "timeout_query": 40 }, "my_processor_4": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 50, "timeout_query": 50 }, "my_processor_5": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 60, "timeout_query": 60 }, "my_processor_6": { "type": "k8s", - "ks8_api_mode": "Manual", + "k8s_api_mode": "Manual", "time_interval": 70, "timeout_query": 70 } From 341effeb7b5e4ed5b73be1f1704e01ed53882809 Mon Sep 17 00:00:00 2001 From: daniel Date: Wed, 23 Aug 2023 11:19:05 +0200 Subject: [PATCH 10/35] refactor: Add new exceptions type for BindingManger --- powerapi/cli/binding_manager.py | 12 ++++++------ powerapi/exception.py | 21 ++++++++++++++++++++- tests/unit/cli/test_binding_manager.py | 6 +++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py index 4c955cfb..6e3bc95e 100644 --- a/powerapi/cli/binding_manager.py +++ b/powerapi/cli/binding_manager.py @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from powerapi.actor import Actor from powerapi.exception import MissingArgumentException, MissingValueException, BadInputData, \ - UnsupportedActorTypeException + UnsupportedActorTypeException, BindingWrongActorsException, UnexistingActorException from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor @@ -134,7 +134,7 @@ def process_bindings(self, bindings: dict): # Check types and do the processing if isinstance(from_actor, ProcessorActor): if not isinstance(to_actor, PusherActor): - raise BadInputData() + raise UnsupportedActorTypeException(actor_type=type(to_actor).__name__) # The processor has to be between the formula and the pusher # The pusher becomes a target of the processor @@ -161,7 +161,7 @@ def process_bindings(self, bindings: dict): elif isinstance(to_actor, ProcessorActor): if not isinstance(from_actor, PullerActor): - raise BadInputData() + raise UnsupportedActorTypeException(actor_type=type(from_actor).__name__) # The processor has to be between the puller and the dispatcher # The dispatcher becomes a target of the processor @@ -180,7 +180,7 @@ def process_bindings(self, bindings: dict): current_filter[1] = processor from_actor.state.report_filter.filters[index] = tuple(current_filter) else: - raise BadInputData() + raise BindingWrongActorsException() def add_actor(self, actor: Actor): """ @@ -208,7 +208,7 @@ def add_actors(self, actors: dict): def check_actor_path(self, actor_path: str): """ - Check that an actor path is separated by PATH_SEPARATOR, that it has to subpaths (group and actor name) + Check that an actor path is separated by PATH_SEPARATOR, that it has two subpaths (group and actor name) and the actor exist in self.actors. It raises a BadInputData exception is these conditions are not respected. Otherwise, it returns the path in a list with two elements """ @@ -216,6 +216,6 @@ def check_actor_path(self, actor_path: str): path = actor_path.split(PATH_SEPARATOR) if len(path) != 2 or path[0] not in self.actors or path[1] not in self.actors[path[0]]: - raise BadInputData() + raise UnexistingActorException(actor_path=actor_path) return path diff --git a/powerapi/exception.py b/powerapi/exception.py index 968d5c2a..0d5b65e9 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -269,7 +269,7 @@ def __init__(self, model_name: str): class InvalidPrefixException(PowerAPIException): """ - Exception raised when attempting to add a new prefix that is a prefix of a existing one or + Exception raised when attempting to add a new prefix that is a prefix of an existing one or vice-versa """ @@ -337,3 +337,22 @@ class MonitorTypeDoesNotExist(PowerAPIException): def __init__(self, monitor_type: str): PowerAPIException.__init__(self) self.monitor_type = monitor_type + + +class UnexistingActorException(PowerAPIException): + """ + Exception raised when an actor (from or to) referenced in a binding does not exist + """ + + def __init__(self, actor_path: str): + PowerAPIException.__init__(self) + self.actor_path = actor_path + + +class BindingWrongActorsException(PowerAPIException): + """ + Exception raised when at least one of the actors in a binding is not a processor + """ + + def __init__(self): + PowerAPIException.__init__(self) \ No newline at end of file diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py index 9f35afed..a8ec1b7a 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/cli/test_binding_manager.py @@ -31,7 +31,7 @@ from powerapi.cli.binding_manager import ProcessorBindingManager, INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, \ BINDING_GROUP from powerapi.dispatcher import DispatcherActor -from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData +from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData, UnexistingActorException from powerapi.processor.processor_actor import ProcessorActor @@ -133,7 +133,7 @@ def test_process_bindings_for_processor_raise_exception_with_wrong_binding_types Test that an exception is raised with a wrong type for the from actor in a binding """ - with pytest.raises(BadInputData): + with pytest.raises(UnsupportedActorTypeException): processor_binding_manager.process_bindings( bindings=pusher_to_processor_wrong_binding_configuration[BINDING_GROUP]) @@ -145,6 +145,6 @@ def test_process_bindings_for_processor_raisse_exception_with_non_existent_pulle Test that an exception is raised with a non-existent puller """ - with pytest.raises(BadInputData): + with pytest.raises(UnexistingActorException): processor_binding_manager.process_bindings( bindings=not_existent_puller_to_processor_configuration[BINDING_GROUP]) From 06899b800d44d67204c2b15efc0d1fc009bc17d7 Mon Sep 17 00:00:00 2001 From: daniel Date: Wed, 23 Aug 2023 15:24:33 +0200 Subject: [PATCH 11/35] feat: Add validation for processors and bindings as well ass related tests --- powerapi/cli/config_validator.py | 49 ++++++++++++- tests/unit/cli/conftest.py | 34 ++++++++- tests/unit/cli/test_config_validator.py | 70 +++++++++++++++++-- ...th_non_existing_puller_configuration.json} | 0 ...th_non_existing_puller_configuration.json} | 0 .../utils/cli/output_input_configuration.json | 22 ++++++ 6 files changed, 166 insertions(+), 9 deletions(-) rename tests/utils/cli/{k8s_processor_binding_with_non_existent_puller_configuration.json => k8s_processor_binding_with_non_existing_puller_configuration.json} (100%) rename tests/utils/cli/{libvirt_processor_binding_with_non_existent_puller_configuration.json => libvirt_processor_binding_with_non_existing_puller_configuration.json} (100%) create mode 100644 tests/utils/cli/output_input_configuration.json diff --git a/powerapi/cli/config_validator.py b/powerapi/cli/config_validator.py index 57fe087b..39f70bc3 100644 --- a/powerapi/cli/config_validator.py +++ b/powerapi/cli/config_validator.py @@ -32,13 +32,15 @@ from typing import Dict -from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException +from powerapi.exception import MissingArgumentException, NotAllowedArgumentValueException, FileDoesNotExistException, \ + UnexistingActorException class ConfigValidator: """ Validate powerapi config and initialize missing default values """ + @staticmethod def validate(config: Dict): """ @@ -66,7 +68,8 @@ def validate(config: Dict): for input_id in config['input']: input_config = config['input'][input_id] if input_config['type'] == 'csv' \ - and ('files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): + and ( + 'files' not in input_config or input_config['files'] is None or len(input_config['files']) == 0): logging.error("no files parameter found for csv input") raise MissingArgumentException(argument_name='files') @@ -79,10 +82,23 @@ def validate(config: Dict): if 'name' not in input_config: input_config['name'] = 'default_puller' + if 'processor' in config and 'binding' not in config: + logging.error("no binding configuration found") + raise MissingArgumentException(argument_name='binding') + elif 'processor' not in config and 'binding' in config: + logging.error("no processor configuration found") + raise MissingArgumentException(argument_name='processor') + ConfigValidator._validate_input(config) + if 'binding' in config: + ConfigValidator._validate_binding(config) + @staticmethod def _validate_input(config: Dict): + """ + Check that csv input type has files that exist + """ for key, input_config in config['input'].items(): if input_config['type'] == 'csv': list_of_files = input_config['files'] @@ -94,3 +110,32 @@ def _validate_input(config: Dict): for file_name in list_of_files: if not os.access(file_name, os.R_OK): raise FileDoesNotExistException(file_name=file_name) + + @staticmethod + def _validate_binding(config: Dict): + """ + Check that defined bindings use existing actors defined by the configuration + """ + for _, binding_infos in config['binding'].items(): + + if 'from' not in binding_infos: + logging.error("no from parameter found for binding") + raise MissingArgumentException(argument_name='from') + + if 'to' not in binding_infos: + logging.error("no to parameter found for binding") + raise MissingArgumentException(argument_name='to') + + # from_info[0] is the subgroup and from_info[1] the actor name + from_infos = binding_infos['from'].split('.') + + if from_infos[0] not in config or from_infos[1] not in config[from_infos[0]]: + logging.error("from actor does not exist") + raise UnexistingActorException(actor_path=binding_infos['from']) + + # to_info[0] is the subgroup and to_info[1] the actor name + to_infos = binding_infos['to'].split('.') + + if to_infos[0] not in config or to_infos[1] not in config[to_infos[0]]: + logging.error("to actor does not exist") + raise UnexistingActorException(actor_path=binding_infos['to']) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 3ee6502b..ec25a104 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -602,6 +602,13 @@ def empty_cli_configuration(): sys.argv = [] +@pytest.fixture +def output_input_configuration(): + """ + Return a dictionary containing bindings with a processor + """ + return load_configuration_from_json_file(file_name='output_input_configuration.json') + @pytest.fixture(params=['puller_to_libvirt_processor_binding_configuration.json', 'puller_to_k8s_processor_binding_configuration.json']) def puller_to_processor_binding_configuration(request): @@ -611,6 +618,29 @@ def puller_to_processor_binding_configuration(request): return load_configuration_from_json_file(file_name=request.param) +@pytest.fixture +def puller_to_processor_config_without_bindings(puller_to_processor_binding_configuration): + """ + Return a configuration with processors but without bindings + """ + + puller_to_processor_binding_configuration.pop('binding') + + return puller_to_processor_binding_configuration + + +@pytest.fixture +def puller_to_processor_config_without_processors(puller_to_processor_binding_configuration): + + """ + Return a configuration with bindings but without processors + """ + + puller_to_processor_binding_configuration.pop('processor') + + return puller_to_processor_binding_configuration + + @pytest.fixture(params=['pusher_to_libvirt_processor_wrong_binding_configuration.json', 'pusher_to_k8s_processor_wrong_binding_configuration.json']) def pusher_to_processor_wrong_binding_configuration(request): @@ -620,8 +650,8 @@ def pusher_to_processor_wrong_binding_configuration(request): return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture(params=['libvirt_processor_binding_with_non_existent_puller_configuration.json', - 'k8s_processor_binding_with_non_existent_puller_configuration.json']) +@pytest.fixture(params=['libvirt_processor_binding_with_non_existing_puller_configuration.json', + 'k8s_processor_binding_with_non_existing_puller_configuration.json']) def not_existent_puller_to_processor_configuration(request): """ Return a dictionary containing bindings with a puller that doesn't exist diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py index 08e9af85..52563fc2 100644 --- a/tests/unit/cli/test_config_validator.py +++ b/tests/unit/cli/test_config_validator.py @@ -29,7 +29,8 @@ import pytest from powerapi.cli import ConfigValidator -from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException +from powerapi.exception import NotAllowedArgumentValueException, MissingArgumentException, FileDoesNotExistException, \ + UnexistingActorException from tests.utils.cli.base_config_parser import load_configuration_from_json_file @@ -41,16 +42,19 @@ def test_config_in_stream_mode_with_csv_input_raise_an_exception(invalid_csv_io_ ConfigValidator.validate(invalid_csv_io_stream_config) -def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, csv_io_postmortem_config): +def test_config_in_postmortem_mode_with_csv_input_is_validated(create_empty_files_from_config, + csv_io_postmortem_config): """ Test that a valid configuration is detected by the ConfigValidator when stream mode is disabled. The files list for the input has to be transformed into a list """ try: - expected_result = load_configuration_from_json_file(file_name='csv_input_output_stream_mode_enabled_configuration.json') + expected_result = load_configuration_from_json_file( + file_name='csv_input_output_stream_mode_enabled_configuration.json') for current_input in expected_result['input']: if expected_result['input'][current_input]['type'] == 'csv': - expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split(',') + expected_result['input'][current_input]['files'] = ( + expected_result['input'][current_input]['files']).split(',') expected_result['stream'] = False @@ -71,7 +75,8 @@ def test_valid_config_postmortem_csv_input_without_optional_arguments_is_validat expected_result = csv_io_postmortem_config_without_optional_arguments.copy() for current_input in expected_result['input']: if expected_result['input'][current_input]['type'] == 'csv': - expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split(',') + expected_result['input'][current_input]['files'] = (expected_result['input'][current_input]['files']).split( + ',') expected_result['input'][current_input]['name'] = 'default_puller' expected_result['input'][current_input]['model'] = 'HWPCReport' expected_result['stream'] = False @@ -111,3 +116,58 @@ def test_config_without_outputs_raise_an_exception(config_without_output): ConfigValidator.validate(config_without_output) assert raised_exception.value.argument_name == 'output' + + +def test_config_with_processor_but_without_binding_raise_an_exception(puller_to_processor_config_without_bindings): + """ + Test that validation of a configuration with processors but without binding raises a + """ + with pytest.raises(MissingArgumentException) as raised_exception: + ConfigValidator.validate(puller_to_processor_config_without_bindings) + + assert raised_exception.value.argument_name == 'binding' + + +def test_config_without_processor_but_with_binding_raise_an_exception(puller_to_processor_config_without_processors): + """ + Test that validation of a configuration without processors but with binding raises a + """ + with pytest.raises(MissingArgumentException) as raised_exception: + ConfigValidator.validate(puller_to_processor_config_without_processors) + + assert raised_exception.value.argument_name == 'processor' + + +def test_config_with_processor_and_binding_with_unexisting_from_actor_raise_an_exception( + not_existent_puller_to_processor_configuration): + """ + Test that validation of a configuration with unexisting actors raise an exception + """ + with pytest.raises(UnexistingActorException) as raised_exception: + ConfigValidator.validate(not_existent_puller_to_processor_configuration) + + assert raised_exception.value.actor_path == not_existent_puller_to_processor_configuration['binding']['b1']['from'] + + +def test_validation_of_correct_configuration_with_processors_and_bindings(output_input_configuration): + """ + Test that a correct configuration with processors and bindings passes the validation + """ + try: + ConfigValidator.validate(output_input_configuration) + except Exception: + assert False + + assert True + + +def test_validation_of_correct_configuration_without_processors_and_bindings(output_input_configuration): + """ + Test that a correct configuration without processors and bindings passes the validation + """ + try: + ConfigValidator.validate(output_input_configuration) + except Exception: + assert False + + assert True diff --git a/tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/k8s_processor_binding_with_non_existing_puller_configuration.json similarity index 100% rename from tests/utils/cli/k8s_processor_binding_with_non_existent_puller_configuration.json rename to tests/utils/cli/k8s_processor_binding_with_non_existing_puller_configuration.json diff --git a/tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json b/tests/utils/cli/libvirt_processor_binding_with_non_existing_puller_configuration.json similarity index 100% rename from tests/utils/cli/libvirt_processor_binding_with_non_existent_puller_configuration.json rename to tests/utils/cli/libvirt_processor_binding_with_non_existing_puller_configuration.json diff --git a/tests/utils/cli/output_input_configuration.json b/tests/utils/cli/output_input_configuration.json new file mode 100644 index 00000000..5c681b90 --- /dev/null +++ b/tests/utils/cli/output_input_configuration.json @@ -0,0 +1,22 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + } +} \ No newline at end of file From c7580d785e722aa917ba428af561ce4dedb2480f Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 24 Aug 2023 10:54:18 +0200 Subject: [PATCH 12/35] fix: Correct subparser k8s processor arguments types and names --- powerapi/cli/common_cli_parsing_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index d410d0e1..bcfafb02 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -450,17 +450,19 @@ def __init__(self): subparser_k8s_processor = SubgroupConfigParsingManager("k8s") subparser_k8s_processor.add_argument( - "a", "api_mode", help_text="k8s api mode (local, manual or cluster)" + "a", "k8s_api_mode", help_text="k8s api mode (local, manual or cluster)" ) subparser_k8s_processor.add_argument( "t", "time_interval", help_text="time interval for the k8s monitoring", + argument_type=int ) subparser_k8s_processor.add_argument( "o", "timeout_query", help_text="timeout for k8s queries", + argument_type=int ) subparser_k8s_processor.add_argument("n", "name", help_text="") self.add_subgroup_parser( From ae20d2be797a4ecc9dbe27e5a43320bf82f16842 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 26 Sep 2023 13:17:54 +0200 Subject: [PATCH 13/35] refactor: Separate pre-processors from post-processors, adapt processors parsing for including bindings to pullers/pushers, remove bindings as first class entities --- powerapi/cli/binding_manager.py | 263 +++++++++--------- powerapi/cli/common_cli_parsing_manager.py | 55 ++-- powerapi/cli/config_validator.py | 40 ++- powerapi/cli/generator.py | 90 ++++-- powerapi/exception.py | 20 +- powerapi/processor/{k8s => pre}/__init__.py | 0 .../{libvirt => pre/k8s}/__init__.py | 0 .../{ => pre}/k8s/k8s_monitor_actor.py | 2 +- .../{ => pre}/k8s/k8s_monitor_handlers.py | 0 .../k8s/k8s_pre_processor_actor.py} | 41 +-- .../k8s/k8s_pre_processor_handlers.py} | 8 +- powerapi/processor/pre/libvirt/__init__.py | 28 ++ .../libvirt/libvirt_pre_processor_actor.py} | 25 +- .../libvirt_pre_processor_handlers.py} | 4 +- powerapi/processor/processor_actor.py | 13 +- tests/unit/cli/conftest.py | 176 +++++++----- tests/unit/cli/test_binding_manager.py | 158 ++++++----- tests/unit/cli/test_config_validator.py | 37 +-- tests/unit/cli/test_generator.py | 130 ++++----- tests/unit/processor/conftest.py | 2 +- tests/unit/processor/{k8s => pre}/__init__.py | 0 .../{libvirt => pre/k8s}/__init__.py | 0 .../{ => pre}/k8s/test_k8s_monitor.py | 2 +- .../{ => pre}/k8s/test_k8s_processor.py | 23 +- tests/unit/processor/pre/libvirt/__init__.py | 28 ++ .../libvirt/test_libvirt_processor.py | 9 +- ...pre_processor_complete_configuration.json} | 12 +- ...n => k8s_pre_processor_configuration.json} | 5 +- ...th_non_existing_puller_configuration.json} | 12 +- ...used_puller_in_bindings_configuration.json | 38 +++ ...rocessor_wrong_binding_configuration.json} | 12 +- ...pre_processor_complete_configuration.json} | 12 +- .../libvirt_pre_processor_configuration.json | 10 + ...th_non_existing_puller_configuration.json} | 18 +- ...used_puller_in_bindings_configuration.json | 36 +++ ...rocessor_wrong_binding_configuration.json} | 12 +- .../cli/libvirt_processor_configuration.json | 9 - ...ral_k8s_pre_processors_configuration.json} | 20 +- ..._without_some_arguments_configuration.json | 28 ++ ..._without_some_arguments_configuration.json | 22 -- ...libvirt_pre_processors_configuration.json} | 20 +- 41 files changed, 846 insertions(+), 574 deletions(-) rename powerapi/processor/{k8s => pre}/__init__.py (100%) rename powerapi/processor/{libvirt => pre/k8s}/__init__.py (100%) rename powerapi/processor/{ => pre}/k8s/k8s_monitor_actor.py (98%) rename powerapi/processor/{ => pre}/k8s/k8s_monitor_handlers.py (100%) rename powerapi/processor/{k8s/k8s_processor_actor.py => pre/k8s/k8s_pre_processor_actor.py} (79%) rename powerapi/processor/{k8s/k8s_processor_handlers.py => pre/k8s/k8s_pre_processor_handlers.py} (95%) create mode 100644 powerapi/processor/pre/libvirt/__init__.py rename powerapi/processor/{libvirt/libvirt_processor_actor.py => pre/libvirt/libvirt_pre_processor_actor.py} (76%) rename powerapi/processor/{libvirt/libvirt_processor_handlers.py => pre/libvirt/libvirt_pre_processor_handlers.py} (96%) rename tests/unit/processor/{k8s => pre}/__init__.py (100%) rename tests/unit/processor/{libvirt => pre/k8s}/__init__.py (100%) rename tests/unit/processor/{ => pre}/k8s/test_k8s_monitor.py (98%) rename tests/unit/processor/{ => pre}/k8s/test_k8s_processor.py (93%) create mode 100644 tests/unit/processor/pre/libvirt/__init__.py rename tests/unit/processor/{ => pre}/libvirt/test_libvirt_processor.py (91%) rename tests/utils/cli/{puller_to_k8s_processor_binding_configuration.json => k8s_pre_processor_complete_configuration.json} (75%) rename tests/utils/cli/{k8s_processor_configuration.json => k8s_pre_processor_configuration.json} (59%) rename tests/utils/cli/{k8s_processor_binding_with_non_existing_puller_configuration.json => k8s_pre_processor_with_non_existing_puller_configuration.json} (74%) create mode 100644 tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json rename tests/utils/cli/{pusher_to_k8s_processor_wrong_binding_configuration.json => k8s_pre_processor_wrong_binding_configuration.json} (75%) rename tests/utils/cli/{puller_to_libvirt_processor_binding_configuration.json => libvirt_pre_processor_complete_configuration.json} (73%) create mode 100644 tests/utils/cli/libvirt_pre_processor_configuration.json rename tests/utils/cli/{libvirt_processor_binding_with_non_existing_puller_configuration.json => libvirt_pre_processor_with_non_existing_puller_configuration.json} (62%) create mode 100644 tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json rename tests/utils/cli/{pusher_to_libvirt_processor_wrong_binding_configuration.json => libvirt_pre_processor_wrong_binding_configuration.json} (73%) delete mode 100644 tests/utils/cli/libvirt_processor_configuration.json rename tests/utils/cli/{several_k8s_processors_configuration.json => several_k8s_pre_processors_configuration.json} (64%) create mode 100644 tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json delete mode 100644 tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json rename tests/utils/cli/{several_libvirt_processors_configuration.json => several_libvirt_pre_processors_configuration.json} (53%) diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py index 6e3bc95e..a36cabf1 100644 --- a/powerapi/cli/binding_manager.py +++ b/powerapi/cli/binding_manager.py @@ -26,21 +26,11 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.actor import Actor -from powerapi.exception import MissingArgumentException, MissingValueException, BadInputData, \ - UnsupportedActorTypeException, BindingWrongActorsException, UnexistingActorException +from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor -FROM_KEY = 'from' -TO_KEY = 'to' -PATH_SEPARATOR = '.' -INPUT_GROUP = 'input' -OUTPUT_GROUP = 'output' -PROCESSOR_GROUP = 'processor' -BINDING_GROUP = 'binding' - class BindingManager: """ @@ -49,109 +39,175 @@ class BindingManager: def __init__(self, actors: dict = {}): """ - :param dict actors: Dictionary of actors to create the bindings + :param dict actors: Dictionary of actors to create the bindings. The name of the actor is the key """ - self.actors = actors + if not actors: + self.actors = {} + else: + self.actors = actors - def process_bindings(self, bindings: dict): + def process_bindings(self): """ - Define bindings between self.actors according to the provided binding dictionary. + Define bindings between self.actors according to the processors' targets. """ raise NotImplementedError() -def check_parameters_in_binding(binding: dict): +class ProcessorBindingManager(BindingManager): """ - Check that a binding has the from and to parameters and that they have a value. - If it is not the case, it raises a MissingArgumentException or + Class for management the binding between processor actors and others actors """ - # Check from and to exist - if FROM_KEY not in binding: - raise MissingArgumentException(argument_name=FROM_KEY) - if TO_KEY not in binding: - raise MissingArgumentException(argument_name=TO_KEY) + def __init__(self, actors: dict, processors: dict): + """ + The ProcessorBindingManager defines bindings between actors and processors + :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ - from_actor_path = binding[FROM_KEY] - to_actor_path = binding[TO_KEY] + BindingManager.__init__(self, actors=actors) + if not processors: + self.processors = {} + else: + self.processors = processors - # Check that from and to values are not empty - if from_actor_path == "" or to_actor_path == "": - raise MissingValueException(argument_name=FROM_KEY + ' or ' + TO_KEY) + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + """ + for target_actor_name in processor.state.target_actors_names: + if target_actor_name not in self.actors: + raise UnexistingActorException(actor=target_actor_name) + def check_processors_targets_are_unique(self): + """ + Check that processors targets are unique, i.e., the same target is not related to + two different processors + """ + used_targets = [] + for _, processor in self.processors.items(): + for target_actor_name in processor.state.target_actors_names: + if target_actor_name in used_targets: + raise TargetActorAlreadyUsed(target_actor=target_actor_name) + else: + used_targets.append(target_actor_name) -class ProcessorBindingManager(BindingManager): + +class PreProcessorBindingManager(ProcessorBindingManager): """ - Class for management the binding between processor actors and others actors + Class for management the binding between pullers and pre-processor actors """ - def __init__(self, actors: dict): + def __init__(self, actors: dict, processors: dict): """ - The ProcessorBindingManager keeps an actor dictionary with the following structure: [][actor_name] - where is 'input' for pullers, 'output' for pushers and - 'processor' for processors + The PreProcessorBindingManager defines bindings between pullers and processors: puller->processor->dispatcher :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} """ - BindingManager.__init__(self, actors={INPUT_GROUP: {}, OUTPUT_GROUP: {}, PROCESSOR_GROUP: {}}) - if actors: - self.add_actors(actors=actors) + ProcessorBindingManager.__init__(self, actors=actors, processors=processors) - def process_bindings(self, bindings: dict): + def process_bindings(self): """ - Define bindings between self.actors according to the provided binding dictionary. - This dictionary has the structure - {"": - "from": "", - "to": "" - } - the "" and "to": "" follow the convention "." - according to the configuration, e.g., "input.my_puller" and "processor.my_libvirt_processor" + Define bindings between self.actors according to the pre-processors' targets. - One of the actors in the binding has to be a processor. If the "to" actor is the processor, the "from" has to be - a puller. If the "from" actor is a processor, the "to" actor has to be a pusher. - :param bindings: The bindings to be processed. """ - for _, binding in bindings.items(): + # Check that processors targets are unique + self.check_processors_targets_are_unique() - check_parameters_in_binding(binding=binding) + # For each processor, we get targets and create the binding: + # puller->processor->disparcher + for _, processor in self.processors.items(): - from_actor_path = binding[FROM_KEY] - to_actor_path = binding[TO_KEY] + self.check_processor_targets(processor=processor) - # Check that the paths have the correct format and the actors with the - # given paths exist + for target_actor_name in processor.state.target_actors_names: - from_actor_path = self.check_actor_path(actor_path=from_actor_path) - to_actor_path = self.check_actor_path(actor_path=to_actor_path) + # The processor has to be between the puller and the dispatcher + # The dispatcher becomes a target of the processor - # Check that actors types are correct + puller_actor = self.actors[target_actor_name] - from_actor = self.actors[from_actor_path[0]][from_actor_path[1]] - to_actor = self.actors[to_actor_path[0]][to_actor_path[1]] + # The dispatcher defines the relationship between the Formula and + # Puller + number_of_filters = len(puller_actor.state.report_filter.filters) + + for index in range(number_of_filters): + # The filters define the relationship with the dispatcher + # The relationship has to be updated + current_filter = list(puller_actor.state.report_filter.filters[index]) + current_filter_dispatcher = current_filter[1] + processor.add_target_actor(actor=current_filter_dispatcher) + current_filter[1] = processor + puller_actor.state.report_filter.filters[index] = tuple(current_filter) - # Check types and do the processing - if isinstance(from_actor, ProcessorActor): - if not isinstance(to_actor, PusherActor): - raise UnsupportedActorTypeException(actor_type=type(to_actor).__name__) + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PullerActor instance + """ + ProcessorBindingManager.check_processor_targets(self, processor=processor) + + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] + + if not isinstance(actor, PullerActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) + + +class PostProcessorBindingManager(ProcessorBindingManager): + """ + Class for management the binding between post-processor and pusher actors + """ + + def __init__(self, actors: dict, processors: dict, pullers: dict): + """ + The PostProcessorBindingManager defines bindings between processors and pushers: formula->processor->pushers + :param dict actors: Dictionary of PusherActors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + ProcessorBindingManager.__init__(self, actors=actors, processors=processors) + self.pullers = pullers + + def process_bindings(self): + """ + Define bindings between self.actors according to the post-processors' targets. + + """ + + # For each processor, we get targets and create the binding: + # formula->processor->pusher + for _, processor in self.processors.items(): + + self.check_processor_targets(processor=processor) + + for target_actor_name in processor.state.target_actors_names: # The processor has to be between the formula and the pusher # The pusher becomes a target of the processor - processor = from_actor - processor.add_target_actor(actor=to_actor) + + pusher_actor = self.actors[target_actor_name] + + processor.add_target_actor(actor=pusher_actor) # We look for the pusher on each dispatcher in order to replace it by - # the processor. - for _, puller in self.actors[INPUT_GROUP]: - for filter in puller.state.report_filter.filters: - dispatcher = filter[1] + # the processor + for _, puller in self.pullers: + + for current_filter in puller.state.report_filter.filters: + dispatcher = current_filter[1] number_of_pushers = len(dispatcher.pusher) pusher_updated = False for index in range(number_of_pushers): - if dispatcher.pusher[index] == to_actor: + if dispatcher.pusher[index] == pusher_actor: dispatcher.pusher[index] = processor pusher_updated = True break @@ -159,63 +215,16 @@ def process_bindings(self, bindings: dict): if pusher_updated: dispatcher.update_state_formula_factory() - elif isinstance(to_actor, ProcessorActor): - if not isinstance(from_actor, PullerActor): - raise UnsupportedActorTypeException(actor_type=type(from_actor).__name__) - - # The processor has to be between the puller and the dispatcher - # The dispatcher becomes a target of the processor - - # The dispatcher defines the relationship between the Formula and - # puller - processor = to_actor - number_of_filters = len(from_actor.state.report_filter.filters) - - for index in range(number_of_filters): - # The filters define the relationship with the dispatcher - # The relationship has to be updated - current_filter = list(from_actor.state.report_filter.filters[index]) - current_filter_dispatcher = current_filter[1] - processor.add_target_actor(actor=current_filter_dispatcher) - current_filter[1] = processor - from_actor.state.report_filter.filters[index] = tuple(current_filter) - else: - raise BindingWrongActorsException() - - def add_actor(self, actor: Actor): - """ - Add the actor to the dictionary of actors according to its type. - Actor has to be PullerActor, PusherActor or ProcessorActor. The key of the actor is its name - """ - group = None - if isinstance(actor, PullerActor): - group = INPUT_GROUP - elif isinstance(actor, PusherActor): - group = OUTPUT_GROUP - elif isinstance(actor, ProcessorActor): - group = PROCESSOR_GROUP - else: - raise UnsupportedActorTypeException(actor_type=str(type(actor))) - - self.actors[group][actor.name] = actor - - def add_actors(self, actors: dict): + def check_processor_targets(self, processor: ProcessorActor): """ - Add the dictionary of actors to the manager dictionary + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PusherActor instance """ - for _, actor in actors.items(): - self.add_actor(actor) - - def check_actor_path(self, actor_path: str): - """ - Check that an actor path is separated by PATH_SEPARATOR, that it has two subpaths (group and actor name) - and the actor exist in self.actors. It raises a BadInputData exception is these conditions are not respected. - Otherwise, it returns the path in a list with two elements - """ - - path = actor_path.split(PATH_SEPARATOR) + ProcessorBindingManager.check_processor_targets(self, processor=processor) - if len(path) != 2 or path[0] not in self.actors or path[1] not in self.actors[path[0]]: - raise UnexistingActorException(actor_path=actor_path) + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] - return path + if not isinstance(actor, PusherActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index bcfafb02..035ebe78 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -37,7 +37,8 @@ POWERAPI_ENVIRONMENT_VARIABLE_PREFIX = 'POWERAPI_' POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'OUTPUT_' POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'INPUT_' -POWERAPI_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PROCESSOR_' +POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PRE_PROCESSOR_' +POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'POST_PROCESSOR_' POWERAPI_BINDING_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'BINDING_' POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'REPORT_MODIFIER_' @@ -75,13 +76,13 @@ def __init__(self): prefix=POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX, help_text="specify a database output : --output database_name ARG1 ARG2 ...") - self.add_subgroup(name='processor', - prefix=POWERAPI_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a processor : --processor processor_name ARG1 ARG2 ...") + self.add_subgroup(name='pre-processor', + prefix=POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a pre-processor : --pre-processor pre_processor_name ARG1 ARG2 ...") - self.add_subgroup(name='binding', - prefix=POWERAPI_BINDING_ENVIRONMENT_VARIABLE_PREFIX, - help_text="specify a binding : --binding binding_name ARG1 ARG2 ...") + self.add_subgroup(name='post-processor', + prefix=POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX, + help_text="specify a post-processor : --post-processor post_processor_name ARG1 ARG2 ...") # Parsers @@ -433,41 +434,55 @@ def __init__(self): subgroup_parser=subparser_influx2_output ) - subparser_libvirt_processor = SubgroupConfigParsingManager("libvirt") - subparser_libvirt_processor.add_argument( + subparser_libvirt_pre_processor = SubgroupConfigParsingManager("libvirt") + subparser_libvirt_pre_processor.add_argument( "u", "uri", help_text="libvirt daemon uri", default_value="" ) - subparser_libvirt_processor.add_argument( + subparser_libvirt_pre_processor.add_argument( "d", "domain_regexp", help_text="regexp used to extract domain from cgroup string", ) - subparser_libvirt_processor.add_argument("n", "name", help_text="") + + subparser_libvirt_pre_processor.add_argument( + "p", + "puller", + help_text="target puller for the pre-processor", + ) + + subparser_libvirt_pre_processor.add_argument("n", "name", help_text="") self.add_subgroup_parser( - subgroup_name="processor", - subgroup_parser=subparser_libvirt_processor + subgroup_name="pre-processor", + subgroup_parser=subparser_libvirt_pre_processor ) - subparser_k8s_processor = SubgroupConfigParsingManager("k8s") - subparser_k8s_processor.add_argument( + subparser_k8s_pre_processor = SubgroupConfigParsingManager("k8s") + subparser_k8s_pre_processor.add_argument( "a", "k8s_api_mode", help_text="k8s api mode (local, manual or cluster)" ) - subparser_k8s_processor.add_argument( + subparser_k8s_pre_processor.add_argument( "t", "time_interval", help_text="time interval for the k8s monitoring", argument_type=int ) - subparser_k8s_processor.add_argument( + subparser_k8s_pre_processor.add_argument( "o", "timeout_query", help_text="timeout for k8s queries", argument_type=int ) - subparser_k8s_processor.add_argument("n", "name", help_text="") + + subparser_k8s_pre_processor.add_argument( + "p", + "puller", + help_text="target puller for the pre-processor", + ) + + subparser_k8s_pre_processor.add_argument("n", "name", help_text="") self.add_subgroup_parser( - subgroup_name="processor", - subgroup_parser=subparser_k8s_processor + subgroup_name="pre-processor", + subgroup_parser=subparser_k8s_pre_processor ) subparser_k8s_processor_binding = SubgroupConfigParsingManager("processor") diff --git a/powerapi/cli/config_validator.py b/powerapi/cli/config_validator.py index 39f70bc3..65b62a07 100644 --- a/powerapi/cli/config_validator.py +++ b/powerapi/cli/config_validator.py @@ -82,17 +82,35 @@ def validate(config: Dict): if 'name' not in input_config: input_config['name'] = 'default_puller' - if 'processor' in config and 'binding' not in config: - logging.error("no binding configuration found") - raise MissingArgumentException(argument_name='binding') - elif 'processor' not in config and 'binding' in config: - logging.error("no processor configuration found") - raise MissingArgumentException(argument_name='processor') + if 'pre-processor' in config: + for pre_processor_id in config['pre-processor']: + pre_processor_config = config['pre-processor'][pre_processor_id] - ConfigValidator._validate_input(config) + if 'puller' not in pre_processor_config: + logging.error("no puller name found for pre-processor " + pre_processor_id) + raise MissingArgumentException(argument_name='puller') + + puller_id = pre_processor_config['puller'] + + if puller_id not in config['input']: + logging.error("puller actor " + puller_id + " does not exist") + raise UnexistingActorException(actor=puller_id) + + elif 'post-processor' in config: + for post_processor_id in config['post-processor']: + post_processor_config = config['post-processor'][post_processor_id] - if 'binding' in config: - ConfigValidator._validate_binding(config) + if 'pusher' not in post_processor_config: + logging.error("no pusher name found for post-processor " + post_processor_id) + raise MissingArgumentException(argument_name='pusher') + + pusher_id = post_processor_config['pusher'] + + if pusher_id not in config['output']: + logging.error("pusher actor " + pusher_id + " does not exist") + raise UnexistingActorException(actor=pusher_id) + + ConfigValidator._validate_input(config) @staticmethod def _validate_input(config: Dict): @@ -131,11 +149,11 @@ def _validate_binding(config: Dict): if from_infos[0] not in config or from_infos[1] not in config[from_infos[0]]: logging.error("from actor does not exist") - raise UnexistingActorException(actor_path=binding_infos['from']) + raise UnexistingActorException(actor=binding_infos['from']) # to_info[0] is the subgroup and to_info[1] the actor name to_infos = binding_infos['to'].split('.') if to_infos[0] not in config or to_infos[1] not in config[to_infos[0]]: logging.error("to actor does not exist") - raise UnexistingActorException(actor_path=binding_infos['to']) + raise UnexistingActorException(actor=binding_infos['to']) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index 8c0f50c2..3f6c25fa 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -37,10 +37,10 @@ from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed, MonitorTypeDoesNotExist from powerapi.filter import Filter -from powerapi.processor.k8s.k8s_monitor_actor import K8sMonitorAgentActor -from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ +from powerapi.processor.pre.k8s.k8s_monitor_actor import K8sMonitorAgentActor +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE -from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor from powerapi.report import HWPCReport, PowerReport, ControlReport, ProcfsReport, Report, FormulaReport from powerapi.database import MongoDB, CsvDB, InfluxDB, OpenTSDB, SocketDB, PrometheusDB, DirectPrometheusDB, \ VirtioFSDB, FileDB @@ -65,6 +65,8 @@ K8S_API_MODE_KEY = 'k8s_api_mode' TIME_INTERVAL_KEY = 'time_interval' TIMEOUT_QUERY_KEY = 'timeout_query' +PULLER_NAME_KEY = 'puller' +PUSHER_NAME_KEY = 'pusher' LISTENER_ACTOR_KEY = 'listener_actor' @@ -365,26 +367,16 @@ class ProcessorGenerator(Generator): Generator that initialises the processor from config """ - def __init__(self): - Generator.__init__(self, component_group_name='processor') - - self.processor_factory = { - 'libvirt': lambda processor_config: LibvirtProcessorActor(name=processor_config[ACTOR_NAME_KEY], - uri=processor_config[COMPONENT_URI_KEY], - regexp=processor_config[REGEXP_KEY]), - 'k8s': lambda processor_config: K8sProcessorActor(name=processor_config[ACTOR_NAME_KEY], - ks8_api_mode=None if - K8S_API_MODE_KEY not in processor_config else - processor_config[K8S_API_MODE_KEY], - time_interval=TIME_INTERVAL_DEFAULT_VALUE if - TIME_INTERVAL_KEY not in processor_config else - processor_config[TIME_INTERVAL_KEY], - timeout_query=TIMEOUT_QUERY_DEFAULT_VALUE if - TIMEOUT_QUERY_KEY not in processor_config - else processor_config[TIMEOUT_QUERY_KEY] - ) + def __init__(self, component_group_name: str): + Generator.__init__(self, component_group_name=component_group_name) - } + self.processor_factory = self._get_default_processor_factories() + + def _get_default_processor_factories(self) -> dict: + """ + Init the factories for this processor generator + """ + raise NotImplementedError def remove_processor_factory(self, processor_type: str): """ @@ -404,10 +396,6 @@ def add_processor_factory(self, processor_type: str, processor_factory_function: def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str): - return self._actor_factory(actor_name, main_config, component_config) - - def _actor_factory(self, actor_name: str, _, component_config: dict): - processor_actor_type = component_config[COMPONENT_TYPE_KEY] if processor_actor_type not in self.processor_factory: @@ -419,6 +407,48 @@ def _actor_factory(self, actor_name: str, _, component_config: dict): return self.processor_factory[processor_actor_type](component_config) +class PreProcessorGenerator(ProcessorGenerator): + """ + Generator that initialises the pre-processor from config + """ + + def __init__(self): + ProcessorGenerator.__init__(self, component_group_name='pre-processor') + + def _get_default_processor_factories(self) -> dict: + return { + 'libvirt': lambda processor_config: LibvirtPreProcessorActor(name=processor_config[ACTOR_NAME_KEY], + uri=processor_config[COMPONENT_URI_KEY], + regexp=processor_config[REGEXP_KEY], + target_actors_names=[processor_config + [PULLER_NAME_KEY]]), + 'k8s': lambda processor_config: K8sPreProcessorActor(name=processor_config[ACTOR_NAME_KEY], + ks8_api_mode=None if + K8S_API_MODE_KEY not in processor_config else + processor_config[K8S_API_MODE_KEY], + time_interval=TIME_INTERVAL_DEFAULT_VALUE if + TIME_INTERVAL_KEY not in processor_config else + processor_config[TIME_INTERVAL_KEY], + timeout_query=TIMEOUT_QUERY_DEFAULT_VALUE if + TIMEOUT_QUERY_KEY not in processor_config + else processor_config[TIMEOUT_QUERY_KEY], + target_actors_names=[processor_config[PULLER_NAME_KEY]] + ) + } + + +class PostProcessorGenerator(ProcessorGenerator): + """ + Generator that initialises the post-processor from config + """ + + def __init__(self): + ProcessorGenerator.__init__(self, component_group_name='pre-processor') + + def _get_default_processor_factories(self) -> dict: + return {} + + class MonitorGenerator(Generator): """ Generator that initialises the monitor by using a K8sProcessorActor @@ -458,8 +488,10 @@ def generate_from_processors(self, processors: dict) -> dict: monitors_config = {MONITOR_KEY: {}} for processor_name, processor in processors.items(): - monitors_config[MONITOR_KEY][processor_name + MONITOR_NAME_SUFFIX] = { - COMPONENT_TYPE_KEY: K8S_COMPONENT_TYPE_VALUE, - LISTENER_ACTOR_KEY: processor} + + if isinstance(processor, K8sPreProcessorActor): + monitors_config[MONITOR_KEY][processor_name + MONITOR_NAME_SUFFIX] = { + COMPONENT_TYPE_KEY: K8S_COMPONENT_TYPE_VALUE, + LISTENER_ACTOR_KEY: processor} return self.generate(main_config=monitors_config) diff --git a/powerapi/exception.py b/powerapi/exception.py index 0d5b65e9..ef5449a9 100644 --- a/powerapi/exception.py +++ b/powerapi/exception.py @@ -341,18 +341,28 @@ def __init__(self, monitor_type: str): class UnexistingActorException(PowerAPIException): """ - Exception raised when an actor (from or to) referenced in a binding does not exist + Exception raised when an actor referenced in a processor does not exist """ - def __init__(self, actor_path: str): + def __init__(self, actor: str): PowerAPIException.__init__(self) - self.actor_path = actor_path + self.actor = actor class BindingWrongActorsException(PowerAPIException): """ - Exception raised when at least one of the actors in a binding is not a processor + Exception raised when at least one of the actors in a binding is not of a given type """ def __init__(self): - PowerAPIException.__init__(self) \ No newline at end of file + PowerAPIException.__init__(self) + + +class TargetActorAlreadyUsed(PowerAPIException): + """ + Exception raised when an actor is used by more than one processor + """ + + def __init__(self, target_actor: str): + PowerAPIException.__init__(self) + self.target_actor = target_actor diff --git a/powerapi/processor/k8s/__init__.py b/powerapi/processor/pre/__init__.py similarity index 100% rename from powerapi/processor/k8s/__init__.py rename to powerapi/processor/pre/__init__.py diff --git a/powerapi/processor/libvirt/__init__.py b/powerapi/processor/pre/k8s/__init__.py similarity index 100% rename from powerapi/processor/libvirt/__init__.py rename to powerapi/processor/pre/k8s/__init__.py diff --git a/powerapi/processor/k8s/k8s_monitor_actor.py b/powerapi/processor/pre/k8s/k8s_monitor_actor.py similarity index 98% rename from powerapi/processor/k8s/k8s_monitor_actor.py rename to powerapi/processor/pre/k8s/k8s_monitor_actor.py index 89962b09..fa830414 100644 --- a/powerapi/processor/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_actor.py @@ -39,7 +39,7 @@ from powerapi.actor import State, Actor from powerapi.message import StartMessage, PoisonPillMessage, K8sPodUpdateMessage -from powerapi.processor.k8s.k8s_monitor_handlers import K8sMonitorAgentStartMessageHandler, \ +from powerapi.processor.pre.k8s.k8s_monitor_handlers import K8sMonitorAgentStartMessageHandler, \ K8sMonitorAgentPoisonPillMessageHandler LOCAL_CONFIG_MODE = "local" diff --git a/powerapi/processor/k8s/k8s_monitor_handlers.py b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py similarity index 100% rename from powerapi/processor/k8s/k8s_monitor_handlers.py rename to powerapi/processor/pre/k8s/k8s_monitor_handlers.py diff --git a/powerapi/processor/k8s/k8s_processor_actor.py b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py similarity index 79% rename from powerapi/processor/k8s/k8s_processor_actor.py rename to powerapi/processor/pre/k8s/k8s_pre_processor_actor.py index 83d9e787..dcf53880 100644 --- a/powerapi/processor/k8s/k8s_processor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py @@ -38,10 +38,10 @@ from powerapi.actor import Actor from powerapi.message import K8sPodUpdateMessage, StartMessage, PoisonPillMessage -from powerapi.processor.k8s.k8s_monitor_actor import ADDED_EVENT, DELETED_EVENT, MODIFIED_EVENT -from powerapi.processor.k8s.k8s_processor_handlers import K8sProcessorActorHWPCReportHandler, \ - K8sProcessorActorK8sPodUpdateMessageHandler, K8sProcessorActorStartMessageHandler, \ - K8sProcessorActorPoisonPillMessageHandler +from powerapi.processor.pre.k8s.k8s_monitor_actor import ADDED_EVENT, DELETED_EVENT, MODIFIED_EVENT +from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import K8sPreProcessorActorHWPCReportHandler, \ + K8sPreProcessorActorK8sPodUpdateMessageHandler, K8sPreProcessorActorStartMessageHandler, \ + K8sPreProcessorActorPoisonPillMessageHandler from powerapi.processor.processor_actor import ProcessorState, ProcessorActor from powerapi.report import HWPCReport @@ -130,44 +130,45 @@ def get_pod_labels(self, namespace: str, pod_name: str) -> Dict[str, str]: return self.pod_labels.get((namespace, pod_name), dict) -class K8sProcessorState(ProcessorState): +class K8sPreProcessorState(ProcessorState): """ State related to a K8SProcessorActor """ - def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, k8s_api_mode: str, - time_interval: int, timeout_query: int): - ProcessorState.__init__(self, actor=actor, target_actors=target_actors) + def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, target_actors_names: list, + k8s_api_mode: str, time_interval: int, timeout_query: int): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) self.metadata_cache = metadata_cache self.k8s_api_mode = k8s_api_mode self.time_interval = time_interval self.timeout_query = timeout_query -class K8sProcessorActor(ProcessorActor): +class K8sPreProcessorActor(ProcessorActor): """ - Processor Actor that modifies reports by adding K8s related metadata + Pre-processor Actor that modifies reports by adding K8s related metadata """ - def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, level_logger: int = logging.WARNING, + def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, target_actors_names: list = None, + level_logger: int = logging.WARNING, timeout: int = 5000, time_interval: int = TIME_INTERVAL_DEFAULT_VALUE, timeout_query: int = TIMEOUT_QUERY_DEFAULT_VALUE): - ProcessorActor.__init__(self, name=name, target_actors=target_actors, level_logger=level_logger, + ProcessorActor.__init__(self, name=name, level_logger=level_logger, timeout=timeout) - self.state = K8sProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), - target_actors=target_actors, - k8s_api_mode=ks8_api_mode, time_interval=time_interval, - timeout_query=timeout_query) + self.state = K8sPreProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), + target_actors=target_actors, + k8s_api_mode=ks8_api_mode, time_interval=time_interval, + timeout_query=timeout_query, target_actors_names=target_actors_names) def setup(self): """ Define HWPCReportMessage handler, StartMessage handler and PoisonPillMessage Handler """ ProcessorActor.setup(self) - self.add_handler(message_type=StartMessage, handler=K8sProcessorActorStartMessageHandler(state=self.state)) - self.add_handler(message_type=HWPCReport, handler=K8sProcessorActorHWPCReportHandler(state=self.state)) + self.add_handler(message_type=StartMessage, handler=K8sPreProcessorActorStartMessageHandler(state=self.state)) + self.add_handler(message_type=HWPCReport, handler=K8sPreProcessorActorHWPCReportHandler(state=self.state)) self.add_handler(message_type=PoisonPillMessage, - handler=K8sProcessorActorPoisonPillMessageHandler(state=self.state)) + handler=K8sPreProcessorActorPoisonPillMessageHandler(state=self.state)) self.add_handler(message_type=K8sPodUpdateMessage, - handler=K8sProcessorActorK8sPodUpdateMessageHandler(state=self.state)) + handler=K8sPreProcessorActorK8sPodUpdateMessageHandler(state=self.state)) diff --git a/powerapi/processor/k8s/k8s_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py similarity index 95% rename from powerapi/processor/k8s/k8s_processor_handlers.py rename to powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index e7e92354..a79a2cc4 100644 --- a/powerapi/processor/k8s/k8s_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -36,7 +36,7 @@ POD_NAME_METADATA_KEY = 'pod_name' -class K8sProcessorActorStartMessageHandler(StartHandler): +class K8sPreProcessorActorStartMessageHandler(StartHandler): """ Start the K8sProcessorActor """ @@ -50,7 +50,7 @@ def initialization(self): actor.connect_data() -class K8sProcessorActorHWPCReportHandler(ProcessorReportHandler): +class K8sPreProcessorActorHWPCReportHandler(ProcessorReportHandler): """ Process the HWPC Reports """ @@ -84,7 +84,7 @@ def handle(self, message: Message): self._send_report(report=message) -class K8sProcessorActorPoisonPillMessageHandler(PoisonPillMessageHandler): +class K8sPreProcessorActorPoisonPillMessageHandler(PoisonPillMessageHandler): """ Stop the K8sProcessorActor """ @@ -107,7 +107,7 @@ def teardown(self, soft=False): # self.state.actor.logger.debug('teardown finished..') -class K8sProcessorActorK8sPodUpdateMessageHandler(Handler): +class K8sPreProcessorActorK8sPodUpdateMessageHandler(Handler): """ Process the K8sPodUpdateMessage """ diff --git a/powerapi/processor/pre/libvirt/__init__.py b/powerapi/processor/pre/libvirt/__init__.py new file mode 100644 index 00000000..f964fff4 --- /dev/null +++ b/powerapi/processor/pre/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/powerapi/processor/libvirt/libvirt_processor_actor.py b/powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py similarity index 76% rename from powerapi/processor/libvirt/libvirt_processor_actor.py rename to powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py index 873ab498..892edc02 100644 --- a/powerapi/processor/libvirt/libvirt_processor_actor.py +++ b/powerapi/processor/pre/libvirt/libvirt_pre_processor_actor.py @@ -31,8 +31,8 @@ from powerapi.exception import LibvirtException from powerapi.message import StartMessage -from powerapi.processor.libvirt.libvirt_processor_handlers import LibvirtProcessorReportHandler, \ - LibvirtProcessorStartHandler +from powerapi.processor.pre.libvirt.libvirt_pre_processor_handlers import LibvirtPreProcessorReportHandler, \ + LibvirtPreProcessorStartHandler from powerapi.report import Report from powerapi.actor import Actor from powerapi.processor.processor_actor import ProcessorActor, ProcessorState @@ -46,34 +46,35 @@ openReadOnly = None -class LibvirtProcessorState(ProcessorState): +class LibvirtPreProcessorState(ProcessorState): """ - State related to a LibvirtProcssorActor + State related to a LibvirtPreProcessorActor """ - def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list): - ProcessorState.__init__(self, actor=actor, target_actors=target_actors) + def __init__(self, actor: Actor, uri: str, regexp: str, target_actors: list, target_actors_names: list): + ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) self.regexp = re.compile(regexp) self.daemon_uri = None if uri == '' else uri self.libvirt = openReadOnly(self.daemon_uri) -class LibvirtProcessorActor(ProcessorActor): +class LibvirtPreProcessorActor(ProcessorActor): """ Processor Actor that modifies reports by replacing libvirt id by open stak uuid """ - def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, + def __init__(self, name: str, uri: str, regexp: str, target_actors: list = None, target_actors_names: list = None, level_logger: int = logging.WARNING, timeout: int = 5000): - ProcessorActor.__init__(self, name=name, target_actors=target_actors, level_logger=level_logger, + ProcessorActor.__init__(self, name=name, level_logger=level_logger, timeout=timeout) - self.state = LibvirtProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors) + self.state = LibvirtPreProcessorState(actor=self, uri=uri, regexp=regexp, target_actors=target_actors, + target_actors_names=target_actors_names) def setup(self): """ Define ReportMessage handler and StartMessage handler """ ProcessorActor.setup(self) - self.add_handler(message_type=StartMessage, handler=LibvirtProcessorStartHandler(state=self.state)) - self.add_handler(message_type=Report, handler=LibvirtProcessorReportHandler(state=self.state)) + self.add_handler(message_type=StartMessage, handler=LibvirtPreProcessorStartHandler(state=self.state)) + self.add_handler(message_type=Report, handler=LibvirtPreProcessorReportHandler(state=self.state)) diff --git a/powerapi/processor/libvirt/libvirt_processor_handlers.py b/powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py similarity index 96% rename from powerapi/processor/libvirt/libvirt_processor_handlers.py rename to powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py index 705f2532..f33cb963 100644 --- a/powerapi/processor/libvirt/libvirt_processor_handlers.py +++ b/powerapi/processor/pre/libvirt/libvirt_pre_processor_handlers.py @@ -43,7 +43,7 @@ libvirtError = LibvirtException -class LibvirtProcessorReportHandler(ProcessorReportHandler): +class LibvirtPreProcessorReportHandler(ProcessorReportHandler): """ Modify reports by replacing libvirt id by open stak uuid """ @@ -69,7 +69,7 @@ def handle(self, report: Report): self._send_report(report=report) -class LibvirtProcessorStartHandler(StartHandler): +class LibvirtPreProcessorStartHandler(StartHandler): """ Initialize the target actors """ diff --git a/powerapi/processor/processor_actor.py b/powerapi/processor/processor_actor.py index 95e3dc15..ca5fba9b 100644 --- a/powerapi/processor/processor_actor.py +++ b/powerapi/processor/processor_actor.py @@ -41,7 +41,7 @@ class ProcessorState(State): - the targets actors """ - def __init__(self, actor: Actor, target_actors: list): + def __init__(self, actor: Actor, target_actors: list, target_actors_names: list): """ :param list target_actors: List of target actors for the processor """ @@ -51,6 +51,7 @@ def __init__(self, actor: Actor, target_actors: list): target_actors = [] self.target_actors = target_actors + self.target_actors_names = target_actors_names class ProcessorActor(Actor): @@ -61,15 +62,9 @@ class ProcessorActor(Actor): actor. """ - def __init__(self, name: str, target_actors: list = None, level_logger: int = logging.WARNING, timeout: int = 5000): - """ - :param list target_actors: Actors that will receive the modified report - """ - + def __init__(self, name: str, level_logger: int = logging.WARNING, timeout: int = 5000): Actor.__init__(self, name, level_logger, timeout) - - #: (State): Actor State. - self.state = ProcessorState(actor=self, target_actors=target_actors) + self.state = ProcessorState(actor=self, target_actors=[], target_actors_names=[]) def setup(self): """ diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index ec25a104..a379a6c0 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -31,9 +31,9 @@ import pytest import tests.utils.cli as test_files_module -from powerapi.cli.binding_manager import INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, ProcessorBindingManager +from powerapi.cli.binding_manager import PreProcessorBindingManager from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator, COMPONENT_TYPE_KEY, \ - LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX + LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX, PreProcessorGenerator from powerapi.dispatcher import DispatcherActor, RouteTable from powerapi.filter import Filter from tests.utils.cli.base_config_parser import load_configuration_from_json_file, \ @@ -188,11 +188,11 @@ def several_inputs_outputs_stream_filedb_without_some_arguments_config(several_i @pytest.fixture -def several_libvirt_processors_config(): +def several_libvirt_pre_processors_config(): """ Configuration with several libvirt processors """ - return load_configuration_from_json_file(file_name='several_libvirt_processors_configuration.json') + return load_configuration_from_json_file(file_name='several_libvirt_pre_processors_configuration.json') @pytest.fixture @@ -205,20 +205,20 @@ def several_libvirt_processors_without_some_arguments_config(): @pytest.fixture -def several_k8s_processors_config(): +def several_k8s_pre_processors_config(): """ Configuration with several k8s processors """ - return load_configuration_from_json_file(file_name='several_k8s_processors_configuration.json') + return load_configuration_from_json_file(file_name='several_k8s_pre_processors_configuration.json') @pytest.fixture -def several_k8s_processors_without_some_arguments_config(): +def several_k8s_pre_processors_without_some_arguments_config(): """ Configuration with several k8s processors """ return load_configuration_from_json_file( - file_name='several_k8s_processors_without_some_arguments_configuration.json') + file_name='several_k8s_pre_processors_without_some_arguments_configuration.json') def generate_k8s_monitors_config(processors_config): @@ -239,25 +239,25 @@ def generate_k8s_monitors_config(processors_config): @pytest.fixture -def several_k8s_processors(several_k8s_processors_config): +def several_k8s_pre_processors(several_k8s_pre_processors_config): """ Return a dictionary with several k8s processors """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(several_k8s_processors_config) + pre_processors = generator.generate(several_k8s_pre_processors_config) - return processors + return pre_processors @pytest.fixture -def several_k8s_monitors_config(several_k8s_processors): +def several_k8s_monitors_config(several_k8s_pre_processors): """ Configuration with several k8s monitors derived from processor generators """ monitors_config = {'monitor': {}} - for processor_name, processor in several_k8s_processors.items(): + for processor_name, processor in several_k8s_pre_processors.items(): monitors_config['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', LISTENER_ACTOR_KEY: processor} @@ -314,40 +314,40 @@ def config_without_output(csv_io_postmortem_config): @pytest.fixture -def libvirt_processor_config(): +def libvirt_pre_processor_config(): """ - Configuration with libvirt as processor + Configuration with libvirt as pre-processor """ - return load_configuration_from_json_file(file_name='libvirt_processor_configuration.json') + return load_configuration_from_json_file(file_name='libvirt_pre_processor_configuration.json') @pytest.fixture -def k8s_processor_config(): +def k8s_pre_processor_config(): """ - Configuration with k8s as processor + Configuration with k8s as pre-processor """ - return load_configuration_from_json_file(file_name='k8s_processor_configuration.json') + return load_configuration_from_json_file(file_name='k8s_pre_processor_configuration.json') @pytest.fixture -def k8s_processors(k8s_processor_config): +def k8s_pre_processors(k8s_pre_processor_config): """ - Return a dictionary of k8s processors + Return a dictionary of k8s pre-processors """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(k8s_processor_config) + processors = generator.generate(k8s_pre_processor_config) return processors @pytest.fixture -def k8s_monitor_config(k8s_processors): +def k8s_monitor_config(k8s_pre_processors): """ The configuration of k8s monitors is derived from processor generators """ - processors = k8s_processors + processors = k8s_pre_processors monitors = {'monitor': {}} @@ -609,78 +609,87 @@ def output_input_configuration(): """ return load_configuration_from_json_file(file_name='output_input_configuration.json') -@pytest.fixture(params=['puller_to_libvirt_processor_binding_configuration.json', - 'puller_to_k8s_processor_binding_configuration.json']) -def puller_to_processor_binding_configuration(request): + +@pytest.fixture(params=['libvirt_pre_processor_complete_configuration.json', + 'k8s_pre_processor_complete_configuration.json']) +def pre_processor_complete_configuration(request): """ - Return a dictionary containing bindings with a processor + Return a dictionary containing a configuration with pre-processor """ return load_configuration_from_json_file(file_name=request.param) @pytest.fixture -def puller_to_processor_config_without_bindings(puller_to_processor_binding_configuration): +def pre_processor_config_without_puller(pre_processor_complete_configuration): """ Return a configuration with processors but without bindings """ - puller_to_processor_binding_configuration.pop('binding') + pre_processor_complete_configuration['pre-processor']['my_processor'].pop('puller') - return puller_to_processor_binding_configuration + return pre_processor_complete_configuration @pytest.fixture -def puller_to_processor_config_without_processors(puller_to_processor_binding_configuration): - +def empty_pre_processor_config(pre_processor_complete_configuration): """ Return a configuration with bindings but without processors """ - puller_to_processor_binding_configuration.pop('processor') + pre_processor_complete_configuration.pop('pre-processor') - return puller_to_processor_binding_configuration + return pre_processor_complete_configuration -@pytest.fixture(params=['pusher_to_libvirt_processor_wrong_binding_configuration.json', - 'pusher_to_k8s_processor_wrong_binding_configuration.json']) -def pusher_to_processor_wrong_binding_configuration(request): +@pytest.fixture(params=['libvirt_pre_processor_wrong_binding_configuration.json', + 'k8s_pre_processor_wrong_binding_configuration.json']) +def pre_processor_wrong_binding_configuration(request): """ - Return a dictionary containing wrong bindings with a processor + Return a dictionary containing wrong bindings with a pre-processor """ return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture(params=['libvirt_processor_binding_with_non_existing_puller_configuration.json', - 'k8s_processor_binding_with_non_existing_puller_configuration.json']) -def not_existent_puller_to_processor_configuration(request): +@pytest.fixture(params=['libvirt_pre_processor_with_non_existing_puller_configuration.json', + 'k8s_pre_processor_with_non_existing_puller_configuration.json']) +def pre_processor_with_unexisting_puller_configuration(request): + """ + Return a dictionary containing a pre-processor with a puller that doesn't exist + """ + return load_configuration_from_json_file( + file_name=request.param) + + +@pytest.fixture(params=['libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json', + 'k8s_pre_processor_with_reused_puller_in_bindings_configuration.json']) +def pre_processor_with_reused_puller_in_bindings_configuration(request): """ - Return a dictionary containing bindings with a puller that doesn't exist + Return a dictionary containing a pre-processor with a puller that doesn't exist """ return load_configuration_from_json_file( file_name=request.param) @pytest.fixture -def processor_binding_actors_and_dictionary(puller_to_processor_binding_configuration): +def pre_processor_pullers_and_processors_dictionaries(pre_processor_complete_configuration): """ - Return a list of dictionary which contains actors as well as the expected unified dictionary related to the actor - list + Return a dictionary which contains puller actors, a dictionary of processors as well as the pushers """ - actors = [] - expected_actors_dictionary = {INPUT_GROUP: {}, OUTPUT_GROUP: {}, PROCESSOR_GROUP: {}} + return get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_complete_configuration) + +def get_pre_processor_pullers_and_processors_dictionaries_from_configuration(configuration: dict) -> (dict, dict, dict): + """ + Return a tuple of dictionaries (pullers, processors) created from the given configuration. + :param dict configuration : Dictionary containing the configuration + """ report_filter = Filter() puller_generator = PullerGenerator(report_filter=report_filter) - pullers = puller_generator.generate(main_config=puller_to_processor_binding_configuration) - actors.append(pullers) - - expected_actors_dictionary[INPUT_GROUP].update(pullers) + pullers = puller_generator.generate(main_config=configuration) pusher_generator = PusherGenerator() - pushers = pusher_generator.generate(main_config=puller_to_processor_binding_configuration) - actors.append(pushers) - - expected_actors_dictionary[OUTPUT_GROUP].update(pushers) + pushers = pusher_generator.generate(main_config=configuration) route_table = RouteTable() @@ -689,13 +698,10 @@ def processor_binding_actors_and_dictionary(puller_to_processor_binding_configur report_filter.filter(lambda msg: True, dispatcher) - processor_generator = ProcessorGenerator() - processors = processor_generator.generate(main_config=puller_to_processor_binding_configuration) - actors.append(processors) - - expected_actors_dictionary[PROCESSOR_GROUP].update(processors) + processor_generator = PreProcessorGenerator() + processors = processor_generator.generate(main_config=configuration) - return actors, expected_actors_dictionary + return pullers, processors, pushers @pytest.fixture @@ -713,13 +719,45 @@ def dispatcher_actor_in_dictionary(): @pytest.fixture -def processor_binding_manager(processor_binding_actors_and_dictionary): +def pre_processor_binding_manager(pre_processor_pullers_and_processors_dictionaries): """ Return a ProcessorBindingManager with a libvirt Processor """ - actors = {} + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] - for current_actors in processor_binding_actors_and_dictionary[0]: - actors.update(current_actors) + return PreProcessorBindingManager(actors=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_binding_configuration): + """ + Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_wrong_binding_configuration) + + return PreProcessorBindingManager(actors=pushers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unexisting_puller_configuration): + """ + Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_unexisting_puller_configuration) + + return PreProcessorBindingManager(actors=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_reused_puller_in_bindings( + pre_processor_with_reused_puller_in_bindings_configuration): + """ + Return a PreProcessorBindingManager with a puller used by two different pre-processors + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_reused_puller_in_bindings_configuration) - return ProcessorBindingManager(actors=actors) + return PreProcessorBindingManager(actors=pullers, processors=processors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py index a8ec1b7a..b958ff4e 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/cli/test_binding_manager.py @@ -1,6 +1,8 @@ # Copyright (c) 2023, INRIA # Copyright (c) 2023, University of Lille # All rights reserved. +import copy + import pytest # Redistribution and use in source and binary forms, with or without @@ -28,123 +30,149 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.cli.binding_manager import ProcessorBindingManager, INPUT_GROUP, OUTPUT_GROUP, PROCESSOR_GROUP, \ - BINDING_GROUP +from powerapi.cli.binding_manager import PreProcessorBindingManager from powerapi.dispatcher import DispatcherActor -from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData, UnexistingActorException +from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData, UnexistingActorException, \ + TargetActorAlreadyUsed from powerapi.processor.processor_actor import ProcessorActor -def check_default_processor_binding_manager_default_actors_content(processor_manager: ProcessorBindingManager): +def check_default_pre_processor_binding_manager_default_actors_content(processor_manager: PreProcessorBindingManager): """ Check the default size for actors dictionary of the manager - :param ProcessorBindingManager processor_manager: Binding manager to check the size + :param PreProcessorBindingManager processor_manager: Binding manager to check the size """ - assert len(processor_manager.actors) == 3 - assert len(processor_manager.actors[INPUT_GROUP]) == 0 - assert len(processor_manager.actors[OUTPUT_GROUP]) == 0 - assert len(processor_manager.actors[PROCESSOR_GROUP]) == 0 + assert len(processor_manager.actors) == 0 + assert len(processor_manager.processors) == 0 -def test_create_processor_binding_manager_with_actors(processor_binding_actors_and_dictionary): +def test_create_pre_processor_binding_manager_with_actors(pre_processor_pullers_and_processors_dictionaries): """ - Test that a ProcessorBindingManager is correctly created when an actor dictionary is provided + Test that a PreProcessorBindingManager is correctly created when an actor and a processor dictionary are provided """ - actors = {} + expected_actors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[0]) + expected_processors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[1]) - for current_actors in processor_binding_actors_and_dictionary[0]: - actors.update(current_actors) - binding_manager = ProcessorBindingManager(actors=actors) + binding_manager = PreProcessorBindingManager(actors=pre_processor_pullers_and_processors_dictionaries[0], + processors=pre_processor_pullers_and_processors_dictionaries[1]) - assert binding_manager.actors == processor_binding_actors_and_dictionary[1] + assert binding_manager.actors == expected_actors_dictionary + assert binding_manager.processors == expected_processors_dictionary def test_create_processor_binding_manager_without_actors(): """ Test that a ProcessorBindingManager is correctly created without a dictionary """ - binding_manager = ProcessorBindingManager(actors=None) + binding_manager = PreProcessorBindingManager(actors=None, processors=None) - check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + check_default_pre_processor_binding_manager_default_actors_content(processor_manager=binding_manager) -def test_create_processor_binding_manager_raise_exception_with_wrong_actor_type(dispatcher_actor_in_dictionary): +def test_process_bindings_for_pre_processor(pre_processor_complete_configuration, + pre_processor_pullers_and_processors_dictionaries): """ - Test that a ProcessorBindingManager is correctly created without a dictionary + Test that the bindings between a puller and a processor are correctly created """ - with pytest.raises(UnsupportedActorTypeException): - _ = ProcessorBindingManager(actors=dispatcher_actor_in_dictionary) + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] + binding_manager = PreProcessorBindingManager(actors=pullers, + processors=processors) -def test_add_actors(processor_binding_actors_and_dictionary): - """ - Test that a dictionary is correctly generated according to a list of actors - """ - binding_manager = ProcessorBindingManager(actors=[]) - - check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], DispatcherActor) - for actors in processor_binding_actors_and_dictionary[0]: - binding_manager.add_actors(actors) + binding_manager.process_bindings() - assert binding_manager.actors == processor_binding_actors_and_dictionary[1] + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], ProcessorActor) + assert pullers['one_puller'].state.report_filter.filters[0][1] == processors['my_processor'] -def test_add_actors_raise_exception_with_wrong_actor_type(dispatcher_actor_in_dictionary): +def test_process_bindings_for_pre_processor_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): """ - Test that a dictionary is correctly generated according to a list of actors + Test that an exception is raised with a wrong type for the from actor in a binding """ - binding_manager = ProcessorBindingManager(actors=[]) - - check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) with pytest.raises(UnsupportedActorTypeException): - binding_manager.add_actors(dispatcher_actor_in_dictionary) + pre_processor_binding_manager_with_wrong_binding_types.process_bindings() + - check_default_processor_binding_manager_default_actors_content(processor_manager=binding_manager) +def test_process_bindings_for_pre_processor_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): + """ + Test that an exception is raised with a puller that doesn't exist + """ + with pytest.raises(UnexistingActorException): + pre_processor_binding_manager_with_unexisting_puller.process_bindings() -def test_process_bindings_for_processor(puller_to_processor_binding_configuration, - processor_binding_actors_and_dictionary): + +def test_process_bindings_for_pre_processor_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): """ - Test that the bindings between a puller and a processor are correctly created + Test that an exception is raised when the same puller is used by several processors """ - actors = {} - for current_actors in processor_binding_actors_and_dictionary[0]: - actors.update(current_actors) + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.process_bindings() - binding_manager = ProcessorBindingManager(actors=actors) - assert len(actors['one_puller'].state.report_filter.filters) == 1 - assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], DispatcherActor) +def test_check_processors_targets_are_unique_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): + """ + Test that an exception is raised when the same puller is used by several processors + """ + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.check_processors_targets_are_unique() - binding_manager.process_bindings(bindings=puller_to_processor_binding_configuration[BINDING_GROUP]) - assert len(actors['one_puller'].state.report_filter.filters) == 1 - assert isinstance(actors['one_puller'].state.report_filter.filters[0][1], ProcessorActor) - assert actors['one_puller'].state.report_filter.filters[0][1] == actors['my_processor'] +def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindings( + pre_processor_binding_manager): + """ + Test that a correct without repeated target passes the validation + """ + try: + pre_processor_binding_manager.check_processors_targets_are_unique() + assert True + except TargetActorAlreadyUsed: + assert False -def test_process_bindings_for_processor_raise_exception_with_wrong_binding_types( - pusher_to_processor_wrong_binding_configuration, - processor_binding_manager): +def test_check_processor_targets_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): """ - Test that an exception is raised with a wrong type for the from actor in a binding + Test that an exception is raised with a puller that doesn't exist """ + pre_processor_binding_manager = pre_processor_binding_manager_with_unexisting_puller + with pytest.raises(UnexistingActorException): + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + +def test_check_processor_targets_raise_exception_with_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): + """ + Test that an exception is raised with a puller that doesn't exist + """ + pre_processor_binding_manager = pre_processor_binding_manager_with_wrong_binding_types with pytest.raises(UnsupportedActorTypeException): - processor_binding_manager.process_bindings( - bindings=pusher_to_processor_wrong_binding_configuration[BINDING_GROUP]) + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) -def test_process_bindings_for_processor_raisse_exception_with_non_existent_puller( - not_existent_puller_to_processor_configuration, - processor_binding_manager): +def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding_manager): """ - Test that an exception is raised with a non-existent puller + Test that validation of a configuration with existing targets of the correct type """ + try: + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) - with pytest.raises(UnexistingActorException): - processor_binding_manager.process_bindings( - bindings=not_existent_puller_to_processor_configuration[BINDING_GROUP]) + assert True + except UnsupportedActorTypeException: + assert False + except UnexistingActorException: + assert False diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py index 52563fc2..4768664d 100644 --- a/tests/unit/cli/test_config_validator.py +++ b/tests/unit/cli/test_config_validator.py @@ -118,52 +118,55 @@ def test_config_without_outputs_raise_an_exception(config_without_output): assert raised_exception.value.argument_name == 'output' -def test_config_with_processor_but_without_binding_raise_an_exception(puller_to_processor_config_without_bindings): +def test_config_with_pre_processor_but_without_puller_raise_an_exception(pre_processor_config_without_puller): """ - Test that validation of a configuration with processors but without binding raises a + Test that validation of a configuration with pre-processors but without a related puller raises a + MissingArgumentException """ with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(puller_to_processor_config_without_bindings) + ConfigValidator.validate(pre_processor_config_without_puller) - assert raised_exception.value.argument_name == 'binding' + assert raised_exception.value.argument_name == 'puller' -def test_config_without_processor_but_with_binding_raise_an_exception(puller_to_processor_config_without_processors): +def test_config_with_empty_pre_processor_pass_validation(empty_pre_processor_config): """ - Test that validation of a configuration without processors but with binding raises a + Test that validation of a configuration without pre-processors passes validation """ - with pytest.raises(MissingArgumentException) as raised_exception: - ConfigValidator.validate(puller_to_processor_config_without_processors) + try: + ConfigValidator.validate(empty_pre_processor_config) - assert raised_exception.value.argument_name == 'processor' + except MissingArgumentException: + assert False -def test_config_with_processor_and_binding_with_unexisting_from_actor_raise_an_exception( - not_existent_puller_to_processor_configuration): +def test_config_with_pre_processor_with_unexisting_puller_actor_raise_an_exception( + pre_processor_with_unexisting_puller_configuration): """ Test that validation of a configuration with unexisting actors raise an exception """ with pytest.raises(UnexistingActorException) as raised_exception: - ConfigValidator.validate(not_existent_puller_to_processor_configuration) + ConfigValidator.validate(pre_processor_with_unexisting_puller_configuration) - assert raised_exception.value.actor_path == not_existent_puller_to_processor_configuration['binding']['b1']['from'] + assert raised_exception.value.actor == \ + pre_processor_with_unexisting_puller_configuration['pre-processor']['my_processor']['puller'] -def test_validation_of_correct_configuration_with_processors_and_bindings(output_input_configuration): +def test_validation_of_correct_configuration_with_pre_processors(pre_processor_complete_configuration): """ Test that a correct configuration with processors and bindings passes the validation """ try: - ConfigValidator.validate(output_input_configuration) + ConfigValidator.validate(pre_processor_complete_configuration) except Exception: assert False assert True -def test_validation_of_correct_configuration_without_processors_and_bindings(output_input_configuration): +def test_validation_of_correct_configuration_without_pre_processors_and_bindings(output_input_configuration): """ - Test that a correct configuration without processors and bindings passes the validation + Test that a correct configuration without pre-processors passes the validation """ try: ConfigValidator.validate(output_input_configuration) diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 198026db..75374bcd 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -34,13 +34,12 @@ import pytest from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator, \ - MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY + MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY, PreProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist -from powerapi.processor.k8s.k8s_monitor_actor import K8sMonitorAgentActor -from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ +from powerapi.processor.pre.k8s.k8s_monitor_actor import K8sMonitorAgentActor +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE -from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor -from powerapi.processor.processor_actor import ProcessorActor +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor from powerapi.database import MongoDB, CsvDB, SocketDB, InfluxDB, PrometheusDB, InfluxDB2 @@ -401,136 +400,143 @@ def test_generate_pusher_when_missing_arguments_in_csv_output_generate_related_a ################################ -# ProcessorActorGenerator Test # +# PreProcessorActorGenerator Test # ################################ -def test_generate_processor_from_empty_config_dict_raise_an_exception(): +def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): """ - Test that ProcessGenerator raise an exception when there is no processor argument + Test that PreProcessGenerator raise an exception when there is no processor argument """ conf = {} - generator = ProcessorGenerator() + generator = PreProcessorGenerator() with pytest.raises(PowerAPIException): generator.generate(conf) -def test_generate_processor_from_libvirt_config(libvirt_processor_config): +def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config): """ - Test that generation for libvirt processor from a config works correctly + Test that generation for libvirt pre-processor from a config works correctly """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(libvirt_processor_config) + processors = generator.generate(libvirt_pre_processor_config) - assert len(processors) == len(libvirt_processor_config) + assert len(processors) == len(libvirt_pre_processor_config) assert 'my_processor' in processors processor = processors['my_processor'] - assert isinstance(processor, LibvirtProcessorActor) + assert isinstance(processor, LibvirtPreProcessorActor) assert processor.state.daemon_uri is None assert isinstance(processor.state.regexp, Pattern) -def test_generate_several_libvirt_processors_from_config(several_libvirt_processors_config): +def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre_processors_config): """ - Test that several libvirt processors are correctly generated + Test that several libvirt pre-processors are correctly generated """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(several_libvirt_processors_config) + processors = generator.generate(several_libvirt_pre_processors_config) - assert len(processors) == len(several_libvirt_processors_config['processor']) + assert len(processors) == len(several_libvirt_pre_processors_config['pre-processor']) - for processor_name, current_processor_infos in several_libvirt_processors_config['processor'].items(): + for processor_name, current_processor_infos in several_libvirt_pre_processors_config['pre-processor'].items(): assert processor_name in processors - assert isinstance(processors[processor_name], LibvirtProcessorActor) + assert isinstance(processors[processor_name], LibvirtPreProcessorActor) assert processors[processor_name].state.daemon_uri is None assert isinstance(processors[processor_name].state.regexp, Pattern) -def test_generate_libvirt_processor_raise_exception_when_missing_arguments( +def test_generate_libvirt_pre_processor_raise_exception_when_missing_arguments( several_libvirt_processors_without_some_arguments_config): """ - Test that ProcessorGenerator raises a PowerAPIException when some arguments are missing for libvirt processor + Test that PreProcessorGenerator raises a PowerAPIException when some arguments are missing for libvirt processor """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() with pytest.raises(PowerAPIException): generator.generate(several_libvirt_processors_without_some_arguments_config) -def check_k8s_processor_infos(processor: K8sProcessorActor, expected_processor_info: dict): +def check_k8s_pre_processor_infos(pre_processor: K8sPreProcessorActor, expected_pre_processor_info: dict): """ Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor """ - assert isinstance(processor, K8sProcessorActor) + assert isinstance(pre_processor, K8sPreProcessorActor) - assert processor.state.k8s_api_mode == expected_processor_info["k8s_api_mode"] - assert processor.state.time_interval == expected_processor_info["time_interval"] - assert processor.state.timeout_query == expected_processor_info["timeout_query"] + assert pre_processor.state.k8s_api_mode == expected_pre_processor_info["k8s_api_mode"] + assert pre_processor.state.time_interval == expected_pre_processor_info["time_interval"] + assert pre_processor.state.timeout_query == expected_pre_processor_info["timeout_query"] + assert len(pre_processor.state.target_actors) == 0 + assert len(pre_processor.state.target_actors_names) == 1 + assert pre_processor.state.target_actors_names[0] == expected_pre_processor_info["puller"] -def test_generate_processor_from_k8s_config(k8s_processor_config): +def test_generate_pre_processor_from_k8s_config(k8s_pre_processor_config): """ Test that generation for k8s processor from a config works correctly """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() processor_name = 'my_processor' - processors = generator.generate(k8s_processor_config) + processors = generator.generate(k8s_pre_processor_config) - assert len(processors) == len(k8s_processor_config) + assert len(processors) == len(k8s_pre_processor_config) assert processor_name in processors processor = processors[processor_name] - check_k8s_processor_infos(processor=processor, - expected_processor_info=k8s_processor_config["processor"][processor_name]) + check_k8s_pre_processor_infos(pre_processor=processor, + expected_pre_processor_info=k8s_pre_processor_config["pre-processor"][processor_name]) -def test_generate_several_k8s_processors_from_config(several_k8s_processors_config): +def test_generate_several_k8s_pre_processors_from_config(several_k8s_pre_processors_config): """ - Test that several k8s processors are correctly generated + Test that several k8s pre-processors are correctly generated """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(several_k8s_processors_config) + processors = generator.generate(several_k8s_pre_processors_config) - assert len(processors) == len(several_k8s_processors_config['processor']) + assert len(processors) == len(several_k8s_pre_processors_config['pre-processor']) - for processor_name, current_processor_infos in several_k8s_processors_config['processor'].items(): + for processor_name, current_processor_infos in several_k8s_pre_processors_config['pre-processor'].items(): assert processor_name in processors processor = processors[processor_name] - check_k8s_processor_infos(processor=processor, expected_processor_info=current_processor_infos) + check_k8s_pre_processor_infos(pre_processor=processor, expected_pre_processor_info=current_processor_infos) -def test_generate_k8s_processor_uses_default_values_with_missing_arguments( - several_k8s_processors_without_some_arguments_config): +def test_generate_k8s_pre_processor_uses_default_values_with_missing_arguments( + several_k8s_pre_processors_without_some_arguments_config): """ - Test that ProcessorGenerator generates a processor with default values when arguments are not defined + Test that PreProcessorGenerator generates a pre-processor with default values when arguments are not defined """ - generator = ProcessorGenerator() + generator = PreProcessorGenerator() - processors = generator.generate(several_k8s_processors_without_some_arguments_config) + processors = generator.generate(several_k8s_pre_processors_without_some_arguments_config) expected_processor_info = {'k8s_api_mode': None, 'time_interval': TIME_INTERVAL_DEFAULT_VALUE, 'timeout_query': TIMEOUT_QUERY_DEFAULT_VALUE} - assert len(processors) == len(several_k8s_processors_without_some_arguments_config['processor']) + assert len(processors) == len(several_k8s_pre_processors_without_some_arguments_config['pre-processor']) - for processor_name in several_k8s_processors_without_some_arguments_config['processor']: - assert processor_name in processors + pre_processor_number = 1 + for pre_processor_name in several_k8s_pre_processors_without_some_arguments_config['pre-processor']: + assert pre_processor_name in processors - processor = processors[processor_name] + processor = processors[pre_processor_name] + expected_processor_info['puller'] = 'my_puller_' + str(pre_processor_number) + + check_k8s_pre_processor_infos(pre_processor=processor, expected_pre_processor_info=expected_processor_info) - check_k8s_processor_infos(processor=processor, expected_processor_info=expected_processor_info) + pre_processor_number += 1 -def check_k8s_monitor_infos(monitor: K8sMonitorAgentActor, associated_processor: K8sProcessorActor): +def check_k8s_monitor_infos(monitor: K8sMonitorAgentActor, associated_processor: K8sPreProcessorActor): """ Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor """ @@ -581,7 +587,7 @@ def test_generate_several_k8s_monitors_from_config(several_k8s_monitors_config): check_k8s_monitor_infos(monitor=monitor, associated_processor=current_monitor_infos[LISTENER_ACTOR_KEY]) -def test_generate_k8s_monitor_from_k8s_processors(k8s_processors): +def test_generate_k8s_monitor_from_k8s_processors(k8s_pre_processors): """ Test that generation for k8s monitor from a processor config works correctly """ @@ -589,29 +595,29 @@ def test_generate_k8s_monitor_from_k8s_processors(k8s_processors): processor_name = 'my_processor' monitor_name = processor_name + MONITOR_NAME_SUFFIX - monitors = generator.generate_from_processors(processors=k8s_processors) + monitors = generator.generate_from_processors(processors=k8s_pre_processors) - assert len(monitors) == len(k8s_processors) + assert len(monitors) == len(k8s_pre_processors) assert monitor_name in monitors monitor = monitors[monitor_name] check_k8s_monitor_infos(monitor=monitor, - associated_processor=k8s_processors[processor_name]) + associated_processor=k8s_pre_processors[processor_name]) -def test_generate_several_k8s_monitors_from_processors(several_k8s_processors): +def test_generate_several_k8s_monitors_from_processors(several_k8s_pre_processors): """ Test that several k8s monitors are correctly generated """ generator = MonitorGenerator() - monitors = generator.generate_from_processors(processors=several_k8s_processors) + monitors = generator.generate_from_processors(processors=several_k8s_pre_processors) - assert len(monitors) == len(several_k8s_processors) + assert len(monitors) == len(several_k8s_pre_processors) - for processor_name, processor in several_k8s_processors.items(): + for processor_name, processor in several_k8s_pre_processors.items(): monitor_name = processor_name + MONITOR_NAME_SUFFIX assert monitor_name in monitors diff --git a/tests/unit/processor/conftest.py b/tests/unit/processor/conftest.py index 8c3d7444..610476af 100644 --- a/tests/unit/processor/conftest.py +++ b/tests/unit/processor/conftest.py @@ -32,7 +32,7 @@ import pytest from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.k8s.k8s_processor_actor import K8sMetadataCache +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sMetadataCache @pytest.fixture(name='pods_list') diff --git a/tests/unit/processor/k8s/__init__.py b/tests/unit/processor/pre/__init__.py similarity index 100% rename from tests/unit/processor/k8s/__init__.py rename to tests/unit/processor/pre/__init__.py diff --git a/tests/unit/processor/libvirt/__init__.py b/tests/unit/processor/pre/k8s/__init__.py similarity index 100% rename from tests/unit/processor/libvirt/__init__.py rename to tests/unit/processor/pre/k8s/__init__.py diff --git a/tests/unit/processor/k8s/test_k8s_monitor.py b/tests/unit/processor/pre/k8s/test_k8s_monitor.py similarity index 98% rename from tests/unit/processor/k8s/test_k8s_monitor.py rename to tests/unit/processor/pre/k8s/test_k8s_monitor.py index 150690d1..3c5c516b 100644 --- a/tests/unit/processor/k8s/test_k8s_monitor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_monitor.py @@ -36,7 +36,7 @@ from kubernetes import client from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.k8s.k8s_monitor_actor import local_config, MANUAL_CONFIG_MODE, \ +from powerapi.processor.pre.k8s.k8s_monitor_actor import local_config, MANUAL_CONFIG_MODE, \ K8sMonitorAgentActor from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe diff --git a/tests/unit/processor/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py similarity index 93% rename from tests/unit/processor/k8s/test_k8s_processor.py rename to tests/unit/processor/pre/k8s/test_k8s_processor.py index fd429849..c3fad7bc 100644 --- a/tests/unit/processor/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -36,9 +36,9 @@ import pytest from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.k8s.k8s_monitor_actor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT -from powerapi.processor.k8s.k8s_processor_actor import K8sProcessorActor -from powerapi.processor.k8s.k8s_processor_handlers import clean_up_container_id, POD_NAMESPACE_METADATA_KEY, \ +from powerapi.processor.pre.k8s.k8s_monitor_actor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor +from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import clean_up_container_id, POD_NAMESPACE_METADATA_KEY, \ POD_NAME_METADATA_KEY from powerapi.report import HWPCReport from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe @@ -185,10 +185,11 @@ def actor(self, started_fake_target_actor, pods_list, mocked_watch_initialized, return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): with patch('kubernetes.config.load_kube_config', return_value=Mock()): with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): - with patch('powerapi.processor.k8s.k8s_processor_actor.K8sMetadataCache', + with patch('powerapi.processor.pre.k8s.k8s_pre_processor_actor.K8sMetadataCache', return_value=multiprocess_metadata_cache_empty): - return K8sProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, - target_actors=[started_fake_target_actor], level_logger=logging.DEBUG) + return K8sPreProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, + target_actors=[started_fake_target_actor], + level_logger=logging.DEBUG) def test_update_metadata_cache_with_added_event(self, started_actor, update_metadata_cache_message_added_event, dummy_pipe_out, shutdown_system): @@ -282,11 +283,11 @@ def test_add_metadata_to_hwpc_report(self, assert result[1] == hwpc_report_with_metadata - def test_add_metadata_to_hwpc_report_does_not_modifie_report_with_unknown_container_id(self, - started_actor, - hwpc_report, - dummy_pipe_out, - shutdown_system): + def test_add_metadata_to_hwpc_report_does_not_modify_report_with_unknown_container_id(self, + started_actor, + hwpc_report, + dummy_pipe_out, + shutdown_system): """ Test that a HWPC report is not modified with an unknown container id """ diff --git a/tests/unit/processor/pre/libvirt/__init__.py b/tests/unit/processor/pre/libvirt/__init__.py new file mode 100644 index 00000000..36c43967 --- /dev/null +++ b/tests/unit/processor/pre/libvirt/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/unit/processor/libvirt/test_libvirt_processor.py b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py similarity index 91% rename from tests/unit/processor/libvirt/test_libvirt_processor.py rename to tests/unit/processor/pre/libvirt/test_libvirt_processor.py index 2d7d9d41..2a731807 100644 --- a/tests/unit/processor/libvirt/test_libvirt_processor.py +++ b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py @@ -33,7 +33,7 @@ import pytest from mock.mock import patch -from powerapi.processor.libvirt.libvirt_processor_actor import LibvirtProcessorActor +from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor from powerapi.report import Report from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe from tests.utils.actor.dummy_actor import DummyActor @@ -63,9 +63,10 @@ def started_fake_dispatcher(self, dummy_pipe_in): @pytest.fixture def actor(self, started_fake_dispatcher): - with patch('powerapi.processor.libvirt.libvirt_processor_actor.openReadOnly', return_value=MockedLibvirt()): - return LibvirtProcessorActor(name='processor_actor', uri='', regexp=REGEXP, - target_actors=[started_fake_dispatcher]) + with patch('powerapi.processor.pre.libvirt.libvirt_pre_processor_actor.openReadOnly', + return_value=MockedLibvirt()): + return LibvirtPreProcessorActor(name='processor_actor', uri='', regexp=REGEXP, + target_actors=[started_fake_dispatcher]) def test_modify_report_that_not_match_regexp_must_not_modify_report(self, started_actor, dummy_pipe_out, diff --git a/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json b/tests/utils/cli/k8s_pre_processor_complete_configuration.json similarity index 75% rename from tests/utils/cli/puller_to_k8s_processor_binding_configuration.json rename to tests/utils/cli/k8s_pre_processor_complete_configuration.json index c3df9832..b637d766 100644 --- a/tests/utils/cli/puller_to_k8s_processor_binding_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_complete_configuration.json @@ -19,19 +19,13 @@ "collection": "my_collection_result" } }, - "processor": { + "pre-processor": { "my_processor": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 50, - "timeout_query": 60 + "timeout_query": 60, + "puller": "one_puller" } - }, - "binding": { - "b1": { - "type": "processor", - "from": "input.one_puller", - "to": "processor.my_processor" - } } } \ No newline at end of file diff --git a/tests/utils/cli/k8s_processor_configuration.json b/tests/utils/cli/k8s_pre_processor_configuration.json similarity index 59% rename from tests/utils/cli/k8s_processor_configuration.json rename to tests/utils/cli/k8s_pre_processor_configuration.json index 29a14636..f82d1732 100644 --- a/tests/utils/cli/k8s_processor_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_configuration.json @@ -1,10 +1,11 @@ { - "processor": { + "pre-processor": { "my_processor": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 20, - "timeout_query": 30 + "timeout_query": 30, + "puller": "my_puller" } } } \ No newline at end of file diff --git a/tests/utils/cli/k8s_processor_binding_with_non_existing_puller_configuration.json b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json similarity index 74% rename from tests/utils/cli/k8s_processor_binding_with_non_existing_puller_configuration.json rename to tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json index fd907375..9a266f3d 100644 --- a/tests/utils/cli/k8s_processor_binding_with_non_existing_puller_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json @@ -19,19 +19,13 @@ "collection": "my_collection_result" } }, - "processor": { + "pre-processor": { "my_processor": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 50, - "timeout_query": 60 + "timeout_query": 60, + "puller": "non_existent_puller" } - }, - "binding": { - "b1": { - "type": "processor", - "from": "input.non_existent_puller", - "to": "processor.my_processor" - } } } \ No newline at end of file diff --git a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json new file mode 100644 index 00000000..ed5918bd --- /dev/null +++ b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json @@ -0,0 +1,38 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "k8s", + "k8s_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 60, + "puller": "one_puller" + }, + "my_processor_2": { + "type": "k8s", + "k8s_api_mode": "Manual", + "time_interval": 50, + "timeout_query": 60, + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json similarity index 75% rename from tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json rename to tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json index d8b2ece6..8751c39d 100644 --- a/tests/utils/cli/pusher_to_k8s_processor_wrong_binding_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json @@ -19,19 +19,13 @@ "collection": "my_collection_result" } }, - "processor": { + "pre-processor": { "my_processor": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 70, - "timeout_query": 80 + "timeout_query": 80, + "puller": "one_pusher" } - }, - "binding": { - "b1": { - "type": "processor", - "from": "output.one_pusher", - "to": "processor.my_processor" - } } } \ No newline at end of file diff --git a/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json b/tests/utils/cli/libvirt_pre_processor_complete_configuration.json similarity index 73% rename from tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json rename to tests/utils/cli/libvirt_pre_processor_complete_configuration.json index a6014e33..a9a0596b 100644 --- a/tests/utils/cli/puller_to_libvirt_processor_binding_configuration.json +++ b/tests/utils/cli/libvirt_pre_processor_complete_configuration.json @@ -19,18 +19,12 @@ "collection": "my_collection_result" } }, - "processor": { + "pre-processor": { "my_processor": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp" + "regexp": "a_reg_exp", + "puller": "one_puller" } - }, - "binding": { - "b1": { - "type": "processor", - "from": "input.one_puller", - "to": "processor.my_processor" - } } } \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_configuration.json b/tests/utils/cli/libvirt_pre_processor_configuration.json new file mode 100644 index 00000000..903d3d16 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_configuration.json @@ -0,0 +1,10 @@ +{ + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "my_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/libvirt_processor_binding_with_non_existing_puller_configuration.json b/tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json similarity index 62% rename from tests/utils/cli/libvirt_processor_binding_with_non_existing_puller_configuration.json rename to tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json index 0a425d37..0f7f9466 100644 --- a/tests/utils/cli/libvirt_processor_binding_with_non_existing_puller_configuration.json +++ b/tests/utils/cli/libvirt_pre_processor_with_non_existing_puller_configuration.json @@ -19,18 +19,12 @@ "collection": "my_collection_result" } }, - "processor": { - "my_processor": { - "type": "libvirt", - "uri": "", - "regexp": "a_reg_exp" - } - }, - "binding": { - "b1": { - "type": "processor", - "from": "input.non_existent_puller", - "to": "processor.my_processor" + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "non_existent_puller" } } } \ No newline at end of file diff --git a/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json new file mode 100644 index 00000000..42e60e29 --- /dev/null +++ b/tests/utils/cli/libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json @@ -0,0 +1,36 @@ +{ + "verbose": true, + "stream": true, + "input": { + "one_puller": { + "model": "HWPCReport", + "type": "mongodb", + "uri": "one_uri", + "db": "my_db", + "collection": "my_collection" + } + }, + "output": { + "one_pusher": { + "type": "mongodb", + "model": "PowerReport", + "uri": "second_uri", + "db": "my_db_result", + "collection": "my_collection_result" + } + }, + "pre-processor": { + "my_processor": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_puller" + }, + "my_processor_2": { + "type": "libvirt", + "uri": "", + "regexp": "a_reg_exp", + "puller": "one_puller" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json b/tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json similarity index 73% rename from tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json rename to tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json index 940c9e2c..9b3a8558 100644 --- a/tests/utils/cli/pusher_to_libvirt_processor_wrong_binding_configuration.json +++ b/tests/utils/cli/libvirt_pre_processor_wrong_binding_configuration.json @@ -19,18 +19,12 @@ "collection": "my_collection_result" } }, - "processor": { + "pre-processor": { "my_processor": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp" + "regexp": "a_reg_exp", + "puller": "one_pusher" } - }, - "binding": { - "b1": { - "type": "processor", - "from": "output.one_pusher", - "to": "processor.my_processor" - } } } \ No newline at end of file diff --git a/tests/utils/cli/libvirt_processor_configuration.json b/tests/utils/cli/libvirt_processor_configuration.json deleted file mode 100644 index d81f5867..00000000 --- a/tests/utils/cli/libvirt_processor_configuration.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "processor": { - "my_processor": { - "type": "libvirt", - "uri": "", - "regexp": "a_reg_exp" - } - } -} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_processors_configuration.json b/tests/utils/cli/several_k8s_pre_processors_configuration.json similarity index 64% rename from tests/utils/cli/several_k8s_processors_configuration.json rename to tests/utils/cli/several_k8s_pre_processors_configuration.json index d56c2eeb..2a7cba50 100644 --- a/tests/utils/cli/several_k8s_processors_configuration.json +++ b/tests/utils/cli/several_k8s_pre_processors_configuration.json @@ -1,40 +1,46 @@ { - "processor": { + "pre-processor": { "my_processor_1": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 20, - "timeout_query": 20 + "timeout_query": 20, + "puller": "my_puller_1" }, "my_processor_2": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 30, - "timeout_query": 30 + "timeout_query": 30, + "puller": "my_puller_2" }, "my_processor_3": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 40, - "timeout_query": 40 + "timeout_query": 40, + "puller": "my_puller_3" }, "my_processor_4": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 50, - "timeout_query": 50 + "timeout_query": 50, + "puller": "my_puller_4" }, "my_processor_5": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 60, - "timeout_query": 60 + "timeout_query": 60, + "puller": "my_puller_5" }, "my_processor_6": { "type": "k8s", "k8s_api_mode": "Manual", "time_interval": 70, - "timeout_query": 70 + "timeout_query": 70, + "puller": "my_puller_6" } } } \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json new file mode 100644 index 00000000..5e51ebad --- /dev/null +++ b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json @@ -0,0 +1,28 @@ +{ + "pre-processor": { + "my_processor_1": { + "type": "k8s", + "puller": "my_puller_1" + }, + "my_processor_2": { + "type": "k8s", + "puller": "my_puller_2" + }, + "my_processor_3": { + "type": "k8s", + "puller": "my_puller_3" + }, + "my_processor_4": { + "type": "k8s", + "puller": "my_puller_4" + }, + "my_processor_5": { + "type": "k8s", + "puller": "my_puller_5" + }, + "my_processor_6": { + "type": "k8s", + "puller": "my_puller_6" + } + } +} \ No newline at end of file diff --git a/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json deleted file mode 100644 index a2729438..00000000 --- a/tests/utils/cli/several_k8s_processors_without_some_arguments_configuration.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "processor": { - "my_processor_1": { - "type": "k8s" - }, - "my_processor_2": { - "type": "k8s" - }, - "my_processor_3": { - "type": "k8s" - }, - "my_processor_4": { - "type": "k8s" - }, - "my_processor_5": { - "type": "k8s" - }, - "my_processor_6": { - "type": "k8s" - } - } -} \ No newline at end of file diff --git a/tests/utils/cli/several_libvirt_processors_configuration.json b/tests/utils/cli/several_libvirt_pre_processors_configuration.json similarity index 53% rename from tests/utils/cli/several_libvirt_processors_configuration.json rename to tests/utils/cli/several_libvirt_pre_processors_configuration.json index 7403a540..4fa686f1 100644 --- a/tests/utils/cli/several_libvirt_processors_configuration.json +++ b/tests/utils/cli/several_libvirt_pre_processors_configuration.json @@ -1,34 +1,40 @@ { - "processor": { + "pre-processor": { "my_processor_1": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_1" + "regexp": "a_reg_exp_1", + "puller": "my_puller_1" }, "my_processor_2": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_2" + "regexp": "a_reg_exp_2", + "puller": "my_puller_2" }, "my_processor_3": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_3" + "regexp": "a_reg_exp_3", + "puller": "my_puller_3" }, "my_processor_4": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_4" + "regexp": "a_reg_exp_4", + "puller": "my_puller_4" }, "my_processor_5": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_5" + "regexp": "a_reg_exp_5", + "puller": "my_puller_5" }, "my_processor_6": { "type": "libvirt", "uri": "", - "regexp": "a_reg_exp_6" + "regexp": "a_reg_exp_6", + "puller": "my_puller_6" } } } \ No newline at end of file From 61d1e7ff5bceaabad2255e24fa2745fecabb85b7 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 26 Sep 2023 16:00:20 +0200 Subject: [PATCH 14/35] refactor: Remove useless code --- powerapi/cli/common_cli_parsing_manager.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index 035ebe78..f00f5f5d 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -485,27 +485,6 @@ def __init__(self): subgroup_parser=subparser_k8s_pre_processor ) - subparser_k8s_processor_binding = SubgroupConfigParsingManager("processor") - - subparser_k8s_processor_binding.add_argument( - "f", - "from", - help_text="starting actor for the binding", - ) - - subparser_k8s_processor_binding.add_argument( - "t", - "to", - help_text="end actor for the binding", - ) - - subparser_k8s_processor_binding.add_argument("n", "name", help_text="") - - self.add_subgroup_parser( - subgroup_name="binding", - subgroup_parser=subparser_k8s_processor_binding - ) - def parse_argv(self): """ """ try: From e6b2d38de95c60ecd5204d14b4a162b7749af14b Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 28 Sep 2023 11:30:03 +0200 Subject: [PATCH 15/35] refactor: Remove useless code --- .../processor/pre/k8s/k8s_monitor_actor.py | 24 +++++++++---------- .../pre/k8s/k8s_pre_processor_handlers.py | 12 ---------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/powerapi/processor/pre/k8s/k8s_monitor_actor.py b/powerapi/processor/pre/k8s/k8s_monitor_actor.py index fa830414..035d238a 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_actor.py @@ -170,7 +170,6 @@ def setup(self): """ Define StartMessage handler and PoisonPillMessage handler """ - print('setup monitor called') self.add_handler(message_type=StartMessage, handler=K8sMonitorAgentStartMessageHandler(state=self.state)) self.add_handler(message_type=PoisonPillMessage, handler=K8sMonitorAgentPoisonPillMessageHandler(state=self.state)) @@ -185,17 +184,18 @@ def query_k8s(self): events = self.k8s_streaming_query(timeout_seconds=self.state.timeout_query, k8sapi_mode=self.state.k8s_api_mode) for event in events: - event_type, namespace, pod_name, container_ids, labels = event - self.state.listener_agent.send_data( - K8sPodUpdateMessage( - sender_name=self.name, - event=event_type, - namespace=namespace, - pod=pod_name, - containers_id=container_ids, - labels=labels + if event: + event_type, namespace, pod_name, container_ids, labels = event + self.state.listener_agent.send_data( + K8sPodUpdateMessage( + sender_name=self.name, + event=event_type, + namespace=namespace, + pod=pod_name, + containers_id=container_ids, + labels=labels + ) ) - ) sleep(self.state.time_interval) except Exception as ex: self.logger.warning(ex) @@ -203,7 +203,7 @@ def query_k8s(self): def k8s_streaming_query(self, timeout_seconds: int, k8sapi_mode: str) -> list: """ - Return a list of events by using the provided paremeters + Return a list of events by using the provided parameters :param int timeout_seconds: Timeout in seconds for waiting for events :param str k8sapi_mode: Kind of API mode """ diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index a79a2cc4..7f7241ea 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -63,8 +63,6 @@ def handle(self, message: Message): # Add pod name, namespace and labels to the report c_id = clean_up_container_id(message.target) - print('c_id', c_id) - print('containers pods', str(self.state.metadata_cache.containers_pod)) namespace, pod = self.state.metadata_cache.get_container_pod(c_id) if namespace is None or pod is None: self.state.actor.logger.warning( @@ -97,16 +95,6 @@ def teardown(self, soft=False): actor.close() -# self.state.actor.logger.debug('Killing monitor actor..') -# self.state.monitor_agent.active_monitoring = False -# self.state.monitor_agent.send_data(PoisonPillMessage(soft=soft, sender_name=self.state.actor.name)) -# if self.state.monitor_agent.is_alive(): -# self.state.monitor_agent.terminate() -# self.state.monitor_agent.socket_interface.close() -# self.state.monitor_agent.join(timeout=10) -# self.state.actor.logger.debug('teardown finished..') - - class K8sPreProcessorActorK8sPodUpdateMessageHandler(Handler): """ Process the K8sPodUpdateMessage From 4e53ad2d61c43662c72dd0925a3eb06fe64e5091 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 19 Oct 2023 14:32:50 +0200 Subject: [PATCH 16/35] refactor: Correct problem related to variable initialisation on K8sMonitorAgentActor, Correct typos, Clean code --- powerapi/actor/actor.py | 2 +- powerapi/cli/common_cli_parsing_manager.py | 8 +-- .../processor/pre/k8s/k8s_monitor_actor.py | 66 ++++++++++--------- .../processor/pre/k8s/k8s_monitor_handlers.py | 5 +- tests/unit/processor/conftest.py | 8 ++- 5 files changed, 47 insertions(+), 42 deletions(-) diff --git a/powerapi/actor/actor.py b/powerapi/actor/actor.py index ea1a70cb..edf9e795 100644 --- a/powerapi/actor/actor.py +++ b/powerapi/actor/actor.py @@ -92,7 +92,7 @@ def __init__(self, name, level_logger=logging.WARNING, timeout=None): :param str name: unique name that will be used to indentify the actor processus :param int level_logger: Define the level of the logger - :param int timeout: if define, do something if no msg is recv every + :param int timeout: if defined, do something if no msg is recv every timeout (in ms) """ multiprocessing.Process.__init__(self, name=name) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index f00f5f5d..11c3865e 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -440,7 +440,7 @@ def __init__(self): ) subparser_libvirt_pre_processor.add_argument( "d", - "domain_regexp", + "domain-regexp", help_text="regexp used to extract domain from cgroup string", ) @@ -458,17 +458,17 @@ def __init__(self): subparser_k8s_pre_processor = SubgroupConfigParsingManager("k8s") subparser_k8s_pre_processor.add_argument( - "a", "k8s_api_mode", help_text="k8s api mode (local, manual or cluster)" + "a", "k8s-api-mode", help_text="k8s api mode (local, manual or cluster)" ) subparser_k8s_pre_processor.add_argument( "t", - "time_interval", + "time-interval", help_text="time interval for the k8s monitoring", argument_type=int ) subparser_k8s_pre_processor.add_argument( "o", - "timeout_query", + "timeout-query", help_text="timeout for k8s queries", argument_type=int ) diff --git a/powerapi/processor/pre/k8s/k8s_monitor_actor.py b/powerapi/processor/pre/k8s/k8s_monitor_actor.py index 035d238a..805e1de5 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_actor.py @@ -31,7 +31,6 @@ import logging from logging import Logger -from time import sleep from kubernetes import client, config, watch from kubernetes.client.configuration import Configuration @@ -180,25 +179,23 @@ def query_k8s(self): """ while self.state.active_monitoring: try: - self.logger.debug("Start - K8sMonitorAgentActor Querying k8s") events = self.k8s_streaming_query(timeout_seconds=self.state.timeout_query, k8sapi_mode=self.state.k8s_api_mode) for event in events: - if event: - event_type, namespace, pod_name, container_ids, labels = event - self.state.listener_agent.send_data( - K8sPodUpdateMessage( - sender_name=self.name, - event=event_type, - namespace=namespace, - pod=pod_name, - containers_id=container_ids, - labels=labels - ) + event_type, namespace, pod_name, container_ids, labels = event + self.state.listener_agent.send_data( + K8sPodUpdateMessage( + sender_name=self.name, + event=event_type, + namespace=namespace, + pod=pod_name, + containers_id=container_ids, + labels=labels ) - sleep(self.state.time_interval) + ) + + # sleep(self.state.time_interval) except Exception as ex: - self.logger.warning(ex) self.logger.warning("Failed streaming query %s", ex) def k8s_streaming_query(self, timeout_seconds: int, k8sapi_mode: str) -> list: @@ -212,30 +209,35 @@ def k8s_streaming_query(self, timeout_seconds: int, k8sapi_mode: str) -> list: w = watch.Watch() try: + event = None for event in w.stream( - api.list_pod_for_all_namespaces, timeout_seconds + func=api.list_pod_for_all_namespaces, timeout_seconds=timeout_seconds ): - if not event["type"] in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: - self.logger.warning( - "UNKNOWN EVENT TYPE : %s : %s %s", - event['type'], event['object'].metadata.name, event + if event: + + if not event["type"] in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: + self.logger.warning( + "UNKNOWN EVENT TYPE : %s : %s %s", + event['type'], event['object'].metadata.name, event + ) + continue + + pod_obj = event["object"] + namespace, pod_name = \ + pod_obj.metadata.namespace, pod_obj.metadata.name + container_ids = ( + [] if event["type"] == "DELETED" + else extract_containers(pod_obj) + ) + labels = pod_obj.metadata.labels + events.append( + (event["type"], namespace, pod_name, container_ids, labels) ) - continue - pod_obj = event["object"] - namespace, pod_name = \ - pod_obj.metadata.namespace, pod_obj.metadata.name - container_ids = ( - [] if event["type"] == "DELETED" - else extract_containers(pod_obj) - ) - labels = pod_obj.metadata.labels - events.append( - (event["type"], namespace, pod_name, container_ids, labels) - ) except ApiException as ae: self.logger.error("APIException %s %s", ae.status, ae) except Exception as undef_e: self.logger.error("Error when watching Exception %s %s", undef_e, event) + return events diff --git a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py index 02c95f15..34ca13a2 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py @@ -43,9 +43,8 @@ def __init__(self, state: State): def initialization(self): self.state.active_monitoring = True self.state.listener_agent.connect_data() - monitoring_thread = Thread(target=self.state.actor.query_k8s) - monitoring_thread.start() - self.state.monitor_thread = monitoring_thread + self.state.monitor_thread = Thread(target=self.state.actor.query_k8s) + self.state.monitor_thread.start() class K8sMonitorAgentPoisonPillMessageHandler(PoisonPillMessageHandler): diff --git a/tests/unit/processor/conftest.py b/tests/unit/processor/conftest.py index 610476af..77699bf9 100644 --- a/tests/unit/processor/conftest.py +++ b/tests/unit/processor/conftest.py @@ -1,7 +1,7 @@ # Copyright (c) 2023, INRIA # Copyright (c) 2023, University of Lille # All rights reserved. - +from typing import Any # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -177,12 +177,16 @@ def __init__(self, events): Mock.__init__(self) self.events = events self.args = None + self.timeout_seconds = 0 + self.func = None - def stream(self, *args): + def stream(self, func: Any, timeout_seconds: int, *args): """ Return the list of events related to the MockedWatch """ self.args = args + self.timeout_seconds = timeout_seconds + self.func = func return self.events From 1bf7ce15925f1dbdeb6d991be2648b5cbd4c47b7 Mon Sep 17 00:00:00 2001 From: daniel Date: Fri, 20 Oct 2023 11:47:40 +0200 Subject: [PATCH 17/35] feat: Add api_key and host as paremeters of K8sPreProcessorActor for manual K8s api configuration --- powerapi/cli/common_cli_parsing_manager.py | 12 ++++++ powerapi/cli/generator.py | 12 +++++- .../processor/pre/k8s/k8s_monitor_actor.py | 41 +++++++++++++------ .../pre/k8s/k8s_pre_processor_actor.py | 11 +++-- .../processor/pre/k8s/test_k8s_monitor.py | 4 +- 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index 11c3865e..a6a63b3b 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -473,6 +473,18 @@ def __init__(self): argument_type=int ) + subparser_k8s_pre_processor.add_argument( + "k", + "api-key", + help_text="API key authorization required for k8s manual configuration", + ) + + subparser_k8s_pre_processor.add_argument( + "h", + "host", + help_text="host required for k8s manual configuration", + ) + subparser_k8s_pre_processor.add_argument( "p", "puller", diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index 3f6c25fa..ea096fc4 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -67,6 +67,8 @@ TIMEOUT_QUERY_KEY = 'timeout_query' PULLER_NAME_KEY = 'puller' PUSHER_NAME_KEY = 'pusher' +API_KEY_KEY = 'api_key' +HOST_KEY = 'host' LISTENER_ACTOR_KEY = 'listener_actor' @@ -432,6 +434,10 @@ def _get_default_processor_factories(self) -> dict: timeout_query=TIMEOUT_QUERY_DEFAULT_VALUE if TIMEOUT_QUERY_KEY not in processor_config else processor_config[TIMEOUT_QUERY_KEY], + api_key=None if API_KEY_KEY not in processor_config + else processor_config[API_KEY_KEY], + host=None if HOST_KEY not in processor_config + else processor_config[HOST_KEY], target_actors_names=[processor_config[PULLER_NAME_KEY]] ) } @@ -451,7 +457,7 @@ def _get_default_processor_factories(self) -> dict: class MonitorGenerator(Generator): """ - Generator that initialises the monitor by using a K8sProcessorActor + Generator that initialises the monitor by using a K8sPreProcessorActor """ def __init__(self): @@ -464,7 +470,9 @@ def __init__(self): k8s_api_mode=monitor_config[LISTENER_ACTOR_KEY].state.k8s_api_mode, time_interval=monitor_config[LISTENER_ACTOR_KEY].state.time_interval, timeout_query=monitor_config[LISTENER_ACTOR_KEY].state.timeout_query, - level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel() + level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel(), + api_key=monitor_config[LISTENER_ACTOR_KEY].state.api_key, + host=monitor_config[LISTENER_ACTOR_KEY].state.host ) } diff --git a/powerapi/processor/pre/k8s/k8s_monitor_actor.py b/powerapi/processor/pre/k8s/k8s_monitor_actor.py index 805e1de5..41190645 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_actor.py @@ -31,6 +31,7 @@ import logging from logging import Logger +from time import sleep from kubernetes import client, config, watch from kubernetes.client.configuration import Configuration @@ -45,12 +46,18 @@ MANUAL_CONFIG_MODE = "manual" CLUSTER_CONFIG_MODE = "cluster" +MANUAL_CONFIG_API_KEY_DEFAULT_VALUE = "YOUR_API_KEY" +MANUAL_CONFIG_HOST_DEFAULT_VALUE = "http://localhost" + ADDED_EVENT = 'ADDED' DELETED_EVENT = 'DELETED' MODIFIED_EVENT = 'MODIFIED' v1_api = None +manual_config_api_key = MANUAL_CONFIG_API_KEY_DEFAULT_VALUE +manual_config_host = MANUAL_CONFIG_HOST_DEFAULT_VALUE + def local_config(): """ @@ -66,9 +73,9 @@ def manual_config(): # Manual config configuration = client.Configuration() # Configure API key authorization: BearerToken - configuration.api_key["authorization"] = "YOUR_API_KEY" + configuration.api_key["authorization"] = manual_config_api_key # Defining host is optional and default to http://localhost - configuration.host = "http://localhost" + configuration.host = manual_config_host Configuration.set_default(configuration) @@ -95,12 +102,18 @@ def load_k8s_client_config(logger: Logger, mode: str = None): }.get(mode, local_config)() -def get_core_v1_api(logger: Logger, mode: str = None): +def get_core_v1_api(logger: Logger, mode: str = None, api_key: str = None, host: str = None): """ Returns a handler to the k8s API. """ global v1_api + global manual_config_api_key + global manual_config_host if v1_api is None: + if api_key: + manual_config_api_key = api_key + if host: + manual_config_host = host load_k8s_client_config(logger=logger, mode=mode) v1_api = client.CoreV1Api() logger.info(f"Core v1 api access : {v1_api}") @@ -135,7 +148,8 @@ class K8sMonitorAgentState(State): State related to a K8sMonitorAgentActor """ - def __init__(self, actor: Actor, time_interval: int, timeout_query: int, listener_agent: Actor, k8s_api_mode: str): + def __init__(self, actor: Actor, time_interval: int, timeout_query: int, listener_agent: Actor, k8s_api_mode: str, + api_key: str = None, host: str = None): State.__init__(self, actor=actor) self.time_interval = time_interval self.timeout_query = timeout_query @@ -143,6 +157,8 @@ def __init__(self, actor: Actor, time_interval: int, timeout_query: int, listene self.k8s_api_mode = k8s_api_mode self.active_monitoring = False self.monitor_thread = None + self.api_key = api_key + self.host = host class K8sMonitorAgentActor(Actor): @@ -152,7 +168,7 @@ class K8sMonitorAgentActor(Actor): """ def __init__(self, name: str, listener_agent: Actor, k8s_api_mode: str = None, time_interval: int = 10, - timeout_query=5, level_logger: int = logging.WARNING): + timeout_query: int = 5, api_key: str = None, host: str = None, level_logger: int = logging.WARNING): """ :param str name: The actor name :param K8sProcessorActor listener_agent: actor waiting for notifications of the monitor @@ -163,7 +179,8 @@ def __init__(self, name: str, listener_agent: Actor, k8s_api_mode: str = None, t """ Actor.__init__(self, name=name, level_logger=level_logger) self.state = K8sMonitorAgentState(actor=self, time_interval=time_interval, timeout_query=timeout_query, - listener_agent=listener_agent, k8s_api_mode=k8s_api_mode) + listener_agent=listener_agent, k8s_api_mode=k8s_api_mode, api_key=api_key, + host=host) def setup(self): """ @@ -179,8 +196,7 @@ def query_k8s(self): """ while self.state.active_monitoring: try: - events = self.k8s_streaming_query(timeout_seconds=self.state.timeout_query, - k8sapi_mode=self.state.k8s_api_mode) + events = self.k8s_streaming_query() for event in events: event_type, namespace, pod_name, container_ids, labels = event self.state.listener_agent.send_data( @@ -194,24 +210,25 @@ def query_k8s(self): ) ) - # sleep(self.state.time_interval) + sleep(self.state.time_interval) except Exception as ex: self.logger.warning("Failed streaming query %s", ex) - def k8s_streaming_query(self, timeout_seconds: int, k8sapi_mode: str) -> list: + def k8s_streaming_query(self) -> list: """ Return a list of events by using the provided parameters :param int timeout_seconds: Timeout in seconds for waiting for events :param str k8sapi_mode: Kind of API mode """ - api = get_core_v1_api(mode=k8sapi_mode, logger=self.logger) + api = get_core_v1_api(mode=self.state.k8s_api_mode, logger=self.logger, api_key=self.state.api_key, + host=self.state.host) events = [] w = watch.Watch() try: event = None for event in w.stream( - func=api.list_pod_for_all_namespaces, timeout_seconds=timeout_seconds + func=api.list_pod_for_all_namespaces, timeout_seconds=self.state.timeout_query ): if event: diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py index dcf53880..c3b416e5 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py @@ -48,7 +48,7 @@ DEFAULT_K8S_CACHE_NAME = 'k8s_cache' DEFAULT_K8S_MONITOR_NAME = 'k8s_monitor' -TIME_INTERVAL_DEFAULT_VALUE = 10 +TIME_INTERVAL_DEFAULT_VALUE = 0 TIMEOUT_QUERY_DEFAULT_VALUE = 5 @@ -136,12 +136,14 @@ class K8sPreProcessorState(ProcessorState): """ def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, target_actors_names: list, - k8s_api_mode: str, time_interval: int, timeout_query: int): + k8s_api_mode: str, time_interval: int, timeout_query: int, api_key: str, host: str): ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) self.metadata_cache = metadata_cache self.k8s_api_mode = k8s_api_mode self.time_interval = time_interval self.timeout_query = timeout_query + self.api_key = api_key + self.host = host class K8sPreProcessorActor(ProcessorActor): @@ -152,14 +154,15 @@ class K8sPreProcessorActor(ProcessorActor): def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, target_actors_names: list = None, level_logger: int = logging.WARNING, timeout: int = 5000, time_interval: int = TIME_INTERVAL_DEFAULT_VALUE, - timeout_query: int = TIMEOUT_QUERY_DEFAULT_VALUE): + timeout_query: int = TIMEOUT_QUERY_DEFAULT_VALUE, api_key: str = None, host: str = None): ProcessorActor.__init__(self, name=name, level_logger=level_logger, timeout=timeout) self.state = K8sPreProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), target_actors=target_actors, k8s_api_mode=ks8_api_mode, time_interval=time_interval, - timeout_query=timeout_query, target_actors_names=target_actors_names) + timeout_query=timeout_query, target_actors_names=target_actors_names, + api_key=api_key, host=host) def setup(self): """ diff --git a/tests/unit/processor/pre/k8s/test_k8s_monitor.py b/tests/unit/processor/pre/k8s/test_k8s_monitor.py index 3c5c516b..78b6579e 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_monitor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_monitor.py @@ -86,7 +86,7 @@ def test_streaming_query(self, started_actor, pods_list, expected_events_list_k8 """ Test that k8s_streaming_query is able to retrieve events related to pods """ - result = started_actor.k8s_streaming_query(timeout_seconds=5, k8sapi_mode=MANUAL_CONFIG_MODE) + result = started_actor.k8s_streaming_query() assert result == expected_events_list_k8s @@ -95,7 +95,7 @@ def test_unknown_events_streaming_query(self, pods_list, mocked_watch_initialize """ Test that unknown events are ignored by k8s_streaming_query """ - result = started_actor.k8s_streaming_query(timeout_seconds=5, k8sapi_mode=MANUAL_CONFIG_MODE) + result = started_actor.k8s_streaming_query() assert result == [] From dfc9bf0ae577bf3141f06623299b0b7e9d44f0fe Mon Sep 17 00:00:00 2001 From: daniel Date: Fri, 20 Oct 2023 12:00:57 +0200 Subject: [PATCH 18/35] style: Update some debug messages --- powerapi/processor/pre/k8s/k8s_monitor_handlers.py | 2 +- powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py index 34ca13a2..1079b72e 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py @@ -56,6 +56,6 @@ def __init__(self, state: State): PoisonPillMessageHandler.__init__(self, state=state) def teardown(self, soft=False): - self.state.actor.logger.debug('teardown monitor') + self.state.actor.logger.debug('teardown K8sMonitorAgent') self.state.active_monitoring = False self.state.monitor_thread.join(10) diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index 7f7241ea..a7ad5751 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -72,7 +72,7 @@ def handle(self, message: Message): message.metadata[POD_NAMESPACE_METADATA_KEY] = namespace message.metadata[POD_NAME_METADATA_KEY] = pod self.state.actor.logger.debug( - f"K8sMdtModifierActor add metadata to report {c_id}, {namespace}, {pod}" + f"K8sPreProcessorActorHWPCReportHandler add metadata to report {c_id}, {namespace}, {pod}" ) labels = self.state.metadata_cache.get_pod_labels(namespace, pod) From 49c53e1653c5bf74096de1dcbd55312b3411a582 Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 30 Oct 2023 09:57:29 +0100 Subject: [PATCH 19/35] refactor: Use multiprocessing.Manager for dealing with Metadata Cache updates in K8sPreProcessorActor, Adapt tests, Remove unnecessary code --- powerapi/cli/binding_manager.py | 230 ------------ powerapi/cli/generator.py | 13 +- powerapi/message.py | 37 -- .../{k8s_monitor_actor.py => k8s_monitor.py} | 112 +++--- .../pre/k8s/k8s_pre_processor_actor.py | 125 ++++--- .../pre/k8s/k8s_pre_processor_handlers.py | 6 +- tests/unit/cli/conftest.py | 47 --- tests/unit/cli/test_binding_manager.py | 178 ---------- tests/unit/cli/test_generator.py | 16 +- tests/unit/processor/conftest.py | 38 +- .../processor/pre/k8s/test_k8s_monitor.py | 108 ++++-- .../processor/pre/k8s/test_k8s_processor.py | 329 +++++++++++------- 12 files changed, 451 insertions(+), 788 deletions(-) delete mode 100644 powerapi/cli/binding_manager.py rename powerapi/processor/pre/k8s/{k8s_monitor_actor.py => k8s_monitor.py} (69%) delete mode 100644 tests/unit/cli/test_binding_manager.py diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py deleted file mode 100644 index a36cabf1..00000000 --- a/powerapi/cli/binding_manager.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright (c) 2023, INRIA -# Copyright (c) 2023, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed -from powerapi.processor.processor_actor import ProcessorActor -from powerapi.puller import PullerActor -from powerapi.pusher import PusherActor - - -class BindingManager: - """ - Class for management the binding between actors during their creation process - """ - - def __init__(self, actors: dict = {}): - """ - :param dict actors: Dictionary of actors to create the bindings. The name of the actor is the key - """ - if not actors: - self.actors = {} - else: - self.actors = actors - - def process_bindings(self): - """ - Define bindings between self.actors according to the processors' targets. - """ - raise NotImplementedError() - - -class ProcessorBindingManager(BindingManager): - """ - Class for management the binding between processor actors and others actors - """ - - def __init__(self, actors: dict, processors: dict): - """ - The ProcessorBindingManager defines bindings between actors and processors - :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} - :param dict processors: Dictionary of processors with structure {:processor1, - :processor2...} - """ - - BindingManager.__init__(self, actors=actors) - if not processors: - self.processors = {} - else: - self.processors = processors - - def check_processor_targets(self, processor: ProcessorActor): - """ - Check that targets of a processor exist in the dictionary of targets. - If it is not the case, it raises a UnexistingActorException - """ - for target_actor_name in processor.state.target_actors_names: - if target_actor_name not in self.actors: - raise UnexistingActorException(actor=target_actor_name) - - def check_processors_targets_are_unique(self): - """ - Check that processors targets are unique, i.e., the same target is not related to - two different processors - """ - used_targets = [] - for _, processor in self.processors.items(): - for target_actor_name in processor.state.target_actors_names: - if target_actor_name in used_targets: - raise TargetActorAlreadyUsed(target_actor=target_actor_name) - else: - used_targets.append(target_actor_name) - - -class PreProcessorBindingManager(ProcessorBindingManager): - """ - Class for management the binding between pullers and pre-processor actors - """ - - def __init__(self, actors: dict, processors: dict): - """ - The PreProcessorBindingManager defines bindings between pullers and processors: puller->processor->dispatcher - :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} - :param dict processors: Dictionary of processors with structure {:processor1, - :processor2...} - """ - - ProcessorBindingManager.__init__(self, actors=actors, processors=processors) - - def process_bindings(self): - """ - Define bindings between self.actors according to the pre-processors' targets. - - """ - - # Check that processors targets are unique - self.check_processors_targets_are_unique() - - # For each processor, we get targets and create the binding: - # puller->processor->disparcher - for _, processor in self.processors.items(): - - self.check_processor_targets(processor=processor) - - for target_actor_name in processor.state.target_actors_names: - - # The processor has to be between the puller and the dispatcher - # The dispatcher becomes a target of the processor - - puller_actor = self.actors[target_actor_name] - - # The dispatcher defines the relationship between the Formula and - # Puller - number_of_filters = len(puller_actor.state.report_filter.filters) - - for index in range(number_of_filters): - # The filters define the relationship with the dispatcher - # The relationship has to be updated - current_filter = list(puller_actor.state.report_filter.filters[index]) - current_filter_dispatcher = current_filter[1] - processor.add_target_actor(actor=current_filter_dispatcher) - current_filter[1] = processor - puller_actor.state.report_filter.filters[index] = tuple(current_filter) - - def check_processor_targets(self, processor: ProcessorActor): - """ - Check that targets of a processor exist in the dictionary of targets. - If it is not the case, it raises a UnexistingActorException - It also checks that the actor is a PullerActor instance - """ - ProcessorBindingManager.check_processor_targets(self, processor=processor) - - for target_actor_name in processor.state.target_actors_names: - actor = self.actors[target_actor_name] - - if not isinstance(actor, PullerActor): - raise UnsupportedActorTypeException(actor_type=type(actor).__name__) - - -class PostProcessorBindingManager(ProcessorBindingManager): - """ - Class for management the binding between post-processor and pusher actors - """ - - def __init__(self, actors: dict, processors: dict, pullers: dict): - """ - The PostProcessorBindingManager defines bindings between processors and pushers: formula->processor->pushers - :param dict actors: Dictionary of PusherActors with structure {:actor1,:actor2...} - :param dict processors: Dictionary of processors with structure {:processor1, - :processor2...} - """ - ProcessorBindingManager.__init__(self, actors=actors, processors=processors) - self.pullers = pullers - - def process_bindings(self): - """ - Define bindings between self.actors according to the post-processors' targets. - - """ - - # For each processor, we get targets and create the binding: - # formula->processor->pusher - for _, processor in self.processors.items(): - - self.check_processor_targets(processor=processor) - - for target_actor_name in processor.state.target_actors_names: - - # The processor has to be between the formula and the pusher - # The pusher becomes a target of the processor - - pusher_actor = self.actors[target_actor_name] - - processor.add_target_actor(actor=pusher_actor) - - # We look for the pusher on each dispatcher in order to replace it by - # the processor - for _, puller in self.pullers: - - for current_filter in puller.state.report_filter.filters: - dispatcher = current_filter[1] - - number_of_pushers = len(dispatcher.pusher) - pusher_updated = False - - for index in range(number_of_pushers): - if dispatcher.pusher[index] == pusher_actor: - dispatcher.pusher[index] = processor - pusher_updated = True - break - - if pusher_updated: - dispatcher.update_state_formula_factory() - - def check_processor_targets(self, processor: ProcessorActor): - """ - Check that targets of a processor exist in the dictionary of targets. - If it is not the case, it raises a UnexistingActorException - It also checks that the actor is a PusherActor instance - """ - ProcessorBindingManager.check_processor_targets(self, processor=processor) - - for target_actor_name in processor.state.target_actors_names: - actor = self.actors[target_actor_name] - - if not isinstance(actor, PusherActor): - raise UnsupportedActorTypeException(actor_type=type(actor).__name__) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index ea096fc4..d0a87c4a 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -37,7 +37,7 @@ from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed, MonitorTypeDoesNotExist from powerapi.filter import Filter -from powerapi.processor.pre.k8s.k8s_monitor_actor import K8sMonitorAgentActor +from powerapi.processor.pre.k8s.k8s_monitor import K8sMonitorAgent from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor @@ -464,15 +464,10 @@ def __init__(self): Generator.__init__(self, component_group_name=MONITOR_KEY) self.monitor_factory = { - K8S_COMPONENT_TYPE_VALUE: lambda monitor_config: K8sMonitorAgentActor( + K8S_COMPONENT_TYPE_VALUE: lambda monitor_config: K8sMonitorAgent( name=monitor_config[ACTOR_NAME_KEY], - listener_agent=monitor_config[LISTENER_ACTOR_KEY], - k8s_api_mode=monitor_config[LISTENER_ACTOR_KEY].state.k8s_api_mode, - time_interval=monitor_config[LISTENER_ACTOR_KEY].state.time_interval, - timeout_query=monitor_config[LISTENER_ACTOR_KEY].state.timeout_query, - level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel(), - api_key=monitor_config[LISTENER_ACTOR_KEY].state.api_key, - host=monitor_config[LISTENER_ACTOR_KEY].state.host + concerned_actor_state=monitor_config[LISTENER_ACTOR_KEY].state, + level_logger=monitor_config[LISTENER_ACTOR_KEY].logger.getEffectiveLevel() ) } diff --git a/powerapi/message.py b/powerapi/message.py index 5952e8a9..1ec910bf 100644 --- a/powerapi/message.py +++ b/powerapi/message.py @@ -169,40 +169,3 @@ def __eq__(self, other): if isinstance(other, PoisonPillMessage): return other.is_soft == self.is_soft and other.is_hard == self.is_hard return False - - -class K8sPodUpdateMessage(Message): - """ - Message sent by the K8sMonitorAgent everytime it detects a change on pod in K8S - """ - - def __init__( - self, - sender_name: str, - event: str, - namespace: str, - pod: str, - containers_id: list[str] = None, - labels: dict[str, str] = None, - ): - """ - :param str sender_name: Name of the sender - :param str event: Event name - :param str namespace: Namespace name - :param str pod: Id of the Pod - :param list containers_id: List of containers id - :param dict labels: Dictionary of labels - """ - Message.__init__(self, sender_name) - self.event = event - self.namespace = namespace - self.pod = pod - self.containers_id = containers_id if containers_id is not None else [] - self.labels = labels if labels is not None else {} - - def __str__(self): - return f"K8sPodUpdateMessage {self.event} {self.namespace} {self.pod}" - - def __eq__(self, message): - return (self.event, self.namespace, self.pod, self.containers_id, self.labels) == \ - (message.event, message.namespace, message.pod, message.containers_id, message.labels) diff --git a/powerapi/processor/pre/k8s/k8s_monitor_actor.py b/powerapi/processor/pre/k8s/k8s_monitor.py similarity index 69% rename from powerapi/processor/pre/k8s/k8s_monitor_actor.py rename to powerapi/processor/pre/k8s/k8s_monitor.py index 41190645..01deff7b 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -30,17 +30,16 @@ # pylint: disable=W0603,W0718 import logging +import multiprocessing from logging import Logger -from time import sleep +from multiprocessing import Process from kubernetes import client, config, watch from kubernetes.client.configuration import Configuration from kubernetes.client.rest import ApiException -from powerapi.actor import State, Actor -from powerapi.message import StartMessage, PoisonPillMessage, K8sPodUpdateMessage -from powerapi.processor.pre.k8s.k8s_monitor_handlers import K8sMonitorAgentStartMessageHandler, \ - K8sMonitorAgentPoisonPillMessageHandler +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorState, K8sMetadataCacheManager, \ + K8sPodUpdateMetadata, DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT LOCAL_CONFIG_MODE = "local" MANUAL_CONFIG_MODE = "manual" @@ -49,10 +48,6 @@ MANUAL_CONFIG_API_KEY_DEFAULT_VALUE = "YOUR_API_KEY" MANUAL_CONFIG_HOST_DEFAULT_VALUE = "http://localhost" -ADDED_EVENT = 'ADDED' -DELETED_EVENT = 'DELETED' -MODIFIED_EVENT = 'MODIFIED' - v1_api = None manual_config_api_key = MANUAL_CONFIG_API_KEY_DEFAULT_VALUE @@ -143,92 +138,87 @@ def extract_containers(pod_obj): return sorted(container_ids) -class K8sMonitorAgentState(State): - """ - State related to a K8sMonitorAgentActor - """ - - def __init__(self, actor: Actor, time_interval: int, timeout_query: int, listener_agent: Actor, k8s_api_mode: str, - api_key: str = None, host: str = None): - State.__init__(self, actor=actor) - self.time_interval = time_interval - self.timeout_query = timeout_query - self.listener_agent = listener_agent - self.k8s_api_mode = k8s_api_mode - self.active_monitoring = False - self.monitor_thread = None - self.api_key = api_key - self.host = host - - -class K8sMonitorAgentActor(Actor): +class K8sMonitorAgent(Process): """ - An actor that monitors the k8s API and sends messages + A monitors the k8s API and sends messages when pod are created, removed or modified. """ - def __init__(self, name: str, listener_agent: Actor, k8s_api_mode: str = None, time_interval: int = 10, - timeout_query: int = 5, api_key: str = None, host: str = None, level_logger: int = logging.WARNING): + def __init__(self, name: str, concerned_actor_state: K8sPreProcessorState, level_logger: int = logging.WARNING): """ :param str name: The actor name - :param K8sProcessorActor listener_agent: actor waiting for notifications of the monitor - :param k8s_api_mode: the used k8s API mode - :param int timeout_query: Timeout for queries - :param int time_interval: Time interval for the monitoring + :param K8sPreProcessorState concerned_actor_state: state of the actor that will use the monitored information :pram int level_logger: The logger level """ - Actor.__init__(self, name=name, level_logger=level_logger) - self.state = K8sMonitorAgentState(actor=self, time_interval=time_interval, timeout_query=timeout_query, - listener_agent=listener_agent, k8s_api_mode=k8s_api_mode, api_key=api_key, - host=host) + Process.__init__(self, name=name) + + #: (logging.Logger): Logger + self.logger = logging.getLogger(name) + self.logger.setLevel(level_logger) + formatter = logging.Formatter('%(asctime)s || %(levelname)s || ' + '%(process)d %(processName)s || %(message)s') + handler = logging.StreamHandler() + handler.setFormatter(formatter) + + # Concerned Actor state + self.concerned_actor_state = concerned_actor_state + + # Multiprocessing Manager + self.manager = multiprocessing.Manager() + + # Shared cache + self.concerned_actor_state.metadata_cache_manager = K8sMetadataCacheManager(process_manager=self.manager, + level_logger=level_logger) - def setup(self): + self.stop_monitoring = self.manager.Event() + + self.stop_monitoring.clear() + + def run(self): """ - Define StartMessage handler and PoisonPillMessage handler + Main code executed by the Monitor """ - self.add_handler(message_type=StartMessage, handler=K8sMonitorAgentStartMessageHandler(state=self.state)) - self.add_handler(message_type=PoisonPillMessage, - handler=K8sMonitorAgentPoisonPillMessageHandler(state=self.state)) + self.query_k8s() def query_k8s(self): """ - Query k8s for changes and send the related information to the listener + Query k8s for changes and update the metadata cache """ - while self.state.active_monitoring: + while not self.stop_monitoring.is_set(): try: events = self.k8s_streaming_query() for event in events: event_type, namespace, pod_name, container_ids, labels = event - self.state.listener_agent.send_data( - K8sPodUpdateMessage( - sender_name=self.name, - event=event_type, - namespace=namespace, - pod=pod_name, - containers_id=container_ids, - labels=labels - ) + + self.concerned_actor_state.metadata_cache_manager.update_cache(metadata=K8sPodUpdateMetadata( + event=event_type, + namespace=namespace, + pod=pod_name, + containers_id=container_ids, + labels=labels + ) ) - sleep(self.state.time_interval) + self.stop_monitoring.wait(timeout=self.concerned_actor_state.time_interval) except Exception as ex: self.logger.warning("Failed streaming query %s", ex) + self.manager.shutdown() + def k8s_streaming_query(self) -> list: """ Return a list of events by using the provided parameters :param int timeout_seconds: Timeout in seconds for waiting for events :param str k8sapi_mode: Kind of API mode """ - api = get_core_v1_api(mode=self.state.k8s_api_mode, logger=self.logger, api_key=self.state.api_key, - host=self.state.host) + api = get_core_v1_api(mode=self.concerned_actor_state.k8s_api_mode, logger=self.logger, + api_key=self.concerned_actor_state.api_key, host=self.concerned_actor_state.host) events = [] w = watch.Watch() try: event = None for event in w.stream( - func=api.list_pod_for_all_namespaces, timeout_seconds=self.state.timeout_query + func=api.list_pod_for_all_namespaces, timeout_seconds=self.concerned_actor_state.timeout_query ): if event: @@ -241,12 +231,15 @@ def k8s_streaming_query(self) -> list: continue pod_obj = event["object"] + namespace, pod_name = \ pod_obj.metadata.namespace, pod_obj.metadata.name + container_ids = ( [] if event["type"] == "DELETED" else extract_containers(pod_obj) ) + labels = pod_obj.metadata.labels events.append( (event["type"], namespace, pod_name, container_ids, labels) @@ -256,5 +249,4 @@ def k8s_streaming_query(self) -> list: self.logger.error("APIException %s %s", ae.status, ae) except Exception as undef_e: self.logger.error("Error when watching Exception %s %s", undef_e, event) - return events diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py index c3b416e5..841907fc 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_actor.py @@ -33,78 +33,118 @@ * `K8sMonitorAgent`, which monitors the k8s API and sends messages when pod are created, removed or modified. """ import logging +from multiprocessing.managers import BaseManager -from typing import Tuple, Dict +from typing import Tuple, Dict, List from powerapi.actor import Actor -from powerapi.message import K8sPodUpdateMessage, StartMessage, PoisonPillMessage -from powerapi.processor.pre.k8s.k8s_monitor_actor import ADDED_EVENT, DELETED_EVENT, MODIFIED_EVENT +from powerapi.message import StartMessage, PoisonPillMessage from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import K8sPreProcessorActorHWPCReportHandler, \ - K8sPreProcessorActorK8sPodUpdateMessageHandler, K8sPreProcessorActorStartMessageHandler, \ - K8sPreProcessorActorPoisonPillMessageHandler + K8sPreProcessorActorStartMessageHandler, K8sPreProcessorActorPoisonPillMessageHandler from powerapi.processor.processor_actor import ProcessorState, ProcessorActor from powerapi.report import HWPCReport -DEFAULT_K8S_CACHE_NAME = 'k8s_cache' +ADDED_EVENT = 'ADDED' +DELETED_EVENT = 'DELETED' +MODIFIED_EVENT = 'MODIFIED' + +DEFAULT_K8S_CACHE_MANAGER_NAME = 'k8s_cache_manager' DEFAULT_K8S_MONITOR_NAME = 'k8s_monitor' +POD_LABELS_KEY = 'pod_labels' +CONTAINERS_POD_KEY = 'containers_pod' +POD_CONTAINERS_KEY = 'pod_containers' + TIME_INTERVAL_DEFAULT_VALUE = 0 TIMEOUT_QUERY_DEFAULT_VALUE = 5 -class K8sMetadataCache: +class K8sPodUpdateMetadata: + """ + Metadata related to a monitored event + """ + + def __init__( + self, + event: str, + namespace: str, + pod: str, + containers_id: List[str] = None, + labels: Dict[str, str] = None, + ): + """ + :param str event: Event name + :param str namespace: Namespace name + :param str pod: Id of the Pod + :param list containers_id: List of containers id + :param dict labels: Dictionary of labels + """ + self.event = event + self.namespace = namespace + self.pod = pod + self.containers_id = containers_id if containers_id is not None else [] + self.labels = labels if labels is not None else {} + + def __str__(self): + return f"K8sPodUpdateMetadata {self.event} {self.namespace} {self.pod}" + + def __eq__(self, metadata): + return (self.event, self.namespace, self.pod, self.containers_id, self.labels) == \ + (metadata.event, metadata.namespace, metadata.pod, metadata.containers_id, metadata.labels) + + +class K8sMetadataCacheManager: """ K8sMetadataCache maintains a cache of pods' metadata (namespace, labels and id of associated containers) """ - def __init__(self, name: str = DEFAULT_K8S_CACHE_NAME, level_logger: int = logging.WARNING): + def __init__(self, process_manager: BaseManager, name: str = DEFAULT_K8S_CACHE_MANAGER_NAME, + level_logger: int = logging.WARNING): + + # Dictionaries - self.pod_labels = {} # (ns, pod) => [labels] - self.containers_pod = {} # container_id => (ns, pod) - self.pod_containers = {} # (ns, pod) => [container_ids] + self.process_manager = process_manager + self.pod_labels = self.process_manager.dict() # (ns, pod) => [labels] + + self.containers_pod = self.process_manager.dict() # container_id => (ns, pod) + + self.pod_containers = self.process_manager.dict() # (ns, pod) => [container_ids] + + # Logger self.logger = logging.getLogger(name) self.logger.setLevel(level_logger) - def update_cache(self, message: K8sPodUpdateMessage): + def update_cache(self, metadata: K8sPodUpdateMetadata): """ Update the local cache for pods. Register this function as a callback for K8sMonitorAgent messages. """ - if message.event == ADDED_EVENT: - self.pod_labels[(message.namespace, message.pod)] = message.labels - self.pod_containers[ - (message.namespace, message.pod) - ] = message.containers_id - for container_id in message.containers_id: + if metadata.event == ADDED_EVENT: + self.pod_labels[(metadata.namespace, metadata.pod)] = metadata.labels + self.pod_containers[(metadata.namespace, metadata.pod)] = metadata.containers_id + for container_id in metadata.containers_id: self.containers_pod[container_id] = \ - (message.namespace, message.pod) + (metadata.namespace, metadata.pod) - elif message.event == DELETED_EVENT: - self.pod_labels.pop((message.namespace, message.pod), None) - for container_id in self.pod_containers.pop( - (message.namespace, message.pod), [] - ): + elif metadata.event == DELETED_EVENT: + self.pod_labels.pop((metadata.namespace, metadata.pod), None) + for container_id in self.pod_containers.pop((metadata.namespace, metadata.pod), []): self.containers_pod.pop(container_id, None) # logger.debug("Pod removed %s %s", message.namespace, message.pod) - elif message.event == MODIFIED_EVENT: - self.pod_labels[(message.namespace, message.pod)] = message.labels - for prev_container_id in self.pod_containers.pop( - (message.namespace, message.pod), [] - ): - self.containers_pod.pop(prev_container_id, None) - self.pod_containers[ - (message.namespace, message.pod) - ] = message.containers_id - for container_id in message.containers_id: - self.containers_pod[container_id] = \ - (message.namespace, message.pod) + elif metadata.event == MODIFIED_EVENT: + self.pod_labels[(metadata.namespace, metadata.pod)] = metadata.labels + for prev_container_id in self.pod_containers.pop((metadata.namespace, metadata.pod), []): + self.pod_containers.pop(prev_container_id, None) + self.pod_containers[(metadata.namespace, metadata.pod)] = metadata.containers_id + for container_id in metadata.containers_id: + self.containers_pod[container_id] = (metadata.namespace, metadata.pod) else: - self.logger.error("Error : unknown event type %s ", message.event) + self.logger.error("Error : unknown event type %s ", metadata.event) def get_container_pod(self, container_id: str) -> Tuple[str, str]: """ @@ -135,10 +175,11 @@ class K8sPreProcessorState(ProcessorState): State related to a K8SProcessorActor """ - def __init__(self, actor: Actor, metadata_cache: K8sMetadataCache, target_actors: list, target_actors_names: list, - k8s_api_mode: str, time_interval: int, timeout_query: int, api_key: str, host: str): + def __init__(self, actor: Actor, target_actors: list, + target_actors_names: list, k8s_api_mode: str, time_interval: int, timeout_query: int, api_key: str, + host: str): ProcessorState.__init__(self, actor=actor, target_actors=target_actors, target_actors_names=target_actors_names) - self.metadata_cache = metadata_cache + self.metadata_cache_manager = None self.k8s_api_mode = k8s_api_mode self.time_interval = time_interval self.timeout_query = timeout_query @@ -158,7 +199,7 @@ def __init__(self, name: str, ks8_api_mode: str, target_actors: list = None, tar ProcessorActor.__init__(self, name=name, level_logger=level_logger, timeout=timeout) - self.state = K8sPreProcessorState(actor=self, metadata_cache=K8sMetadataCache(level_logger=level_logger), + self.state = K8sPreProcessorState(actor=self, target_actors=target_actors, k8s_api_mode=ks8_api_mode, time_interval=time_interval, timeout_query=timeout_query, target_actors_names=target_actors_names, @@ -173,5 +214,3 @@ def setup(self): self.add_handler(message_type=HWPCReport, handler=K8sPreProcessorActorHWPCReportHandler(state=self.state)) self.add_handler(message_type=PoisonPillMessage, handler=K8sPreProcessorActorPoisonPillMessageHandler(state=self.state)) - self.add_handler(message_type=K8sPodUpdateMessage, - handler=K8sPreProcessorActorK8sPodUpdateMessageHandler(state=self.state)) diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index a7ad5751..39cf2fa4 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -63,7 +63,7 @@ def handle(self, message: Message): # Add pod name, namespace and labels to the report c_id = clean_up_container_id(message.target) - namespace, pod = self.state.metadata_cache.get_container_pod(c_id) + namespace, pod = self.state.metadata_cache_manager.get_container_pod(c_id) if namespace is None or pod is None: self.state.actor.logger.warning( f"Container with no associated pod : {message.target}, {c_id}, {namespace}, {pod}" @@ -75,7 +75,7 @@ def handle(self, message: Message): f"K8sPreProcessorActorHWPCReportHandler add metadata to report {c_id}, {namespace}, {pod}" ) - labels = self.state.metadata_cache.get_pod_labels(namespace, pod) + labels = self.state.metadata_cache_manager.get_pod_labels(namespace, pod) for label_key, label_value in labels.items(): message.metadata[f"label_{label_key}"] = label_value @@ -105,7 +105,7 @@ def __init__(self, state: State): def handle(self, message: Message): self.state.actor.logger.debug(f"received K8sPodUpdateMessage message {message}") - self.state.metadata_cache.update_cache(message) + self.state.metadata_cache_manager.update_cache(message) def clean_up_container_id(c_id): diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index a379a6c0..786fb058 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -31,7 +31,6 @@ import pytest import tests.utils.cli as test_files_module -from powerapi.cli.binding_manager import PreProcessorBindingManager from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator, COMPONENT_TYPE_KEY, \ LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX, PreProcessorGenerator from powerapi.dispatcher import DispatcherActor, RouteTable @@ -582,7 +581,6 @@ def cli_configuration(config_file: str): """ Load in sys.argv a configuration with arguments extracted from a json file """ - # config_file = 'root_manager_basic_configuration.json' sys.argv = generate_cli_configuration_from_json_file(file_name=config_file) yield None @@ -716,48 +714,3 @@ def dispatcher_actor_in_dictionary(): route_table=route_table) return {dispatcher.name: dispatcher} - - -@pytest.fixture -def pre_processor_binding_manager(pre_processor_pullers_and_processors_dictionaries): - """ - Return a ProcessorBindingManager with a libvirt Processor - """ - pullers = pre_processor_pullers_and_processors_dictionaries[0] - processors = pre_processor_pullers_and_processors_dictionaries[1] - - return PreProcessorBindingManager(actors=pullers, processors=processors) - - -@pytest.fixture -def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_binding_configuration): - """ - Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) - """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_wrong_binding_configuration) - - return PreProcessorBindingManager(actors=pushers, processors=processors) - - -@pytest.fixture -def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unexisting_puller_configuration): - """ - Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) - """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_with_unexisting_puller_configuration) - - return PreProcessorBindingManager(actors=pullers, processors=processors) - - -@pytest.fixture -def pre_processor_binding_manager_with_reused_puller_in_bindings( - pre_processor_with_reused_puller_in_bindings_configuration): - """ - Return a PreProcessorBindingManager with a puller used by two different pre-processors - """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_with_reused_puller_in_bindings_configuration) - - return PreProcessorBindingManager(actors=pullers, processors=processors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py deleted file mode 100644 index b958ff4e..00000000 --- a/tests/unit/cli/test_binding_manager.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright (c) 2023, INRIA -# Copyright (c) 2023, University of Lille -# All rights reserved. -import copy - -import pytest - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.cli.binding_manager import PreProcessorBindingManager -from powerapi.dispatcher import DispatcherActor -from powerapi.exception import PowerAPIException, UnsupportedActorTypeException, BadInputData, UnexistingActorException, \ - TargetActorAlreadyUsed -from powerapi.processor.processor_actor import ProcessorActor - - -def check_default_pre_processor_binding_manager_default_actors_content(processor_manager: PreProcessorBindingManager): - """ - Check the default size for actors dictionary of the manager - :param PreProcessorBindingManager processor_manager: Binding manager to check the size - """ - assert len(processor_manager.actors) == 0 - assert len(processor_manager.processors) == 0 - - -def test_create_pre_processor_binding_manager_with_actors(pre_processor_pullers_and_processors_dictionaries): - """ - Test that a PreProcessorBindingManager is correctly created when an actor and a processor dictionary are provided - """ - expected_actors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[0]) - expected_processors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[1]) - - binding_manager = PreProcessorBindingManager(actors=pre_processor_pullers_and_processors_dictionaries[0], - processors=pre_processor_pullers_and_processors_dictionaries[1]) - - assert binding_manager.actors == expected_actors_dictionary - assert binding_manager.processors == expected_processors_dictionary - - -def test_create_processor_binding_manager_without_actors(): - """ - Test that a ProcessorBindingManager is correctly created without a dictionary - """ - binding_manager = PreProcessorBindingManager(actors=None, processors=None) - - check_default_pre_processor_binding_manager_default_actors_content(processor_manager=binding_manager) - - -def test_process_bindings_for_pre_processor(pre_processor_complete_configuration, - pre_processor_pullers_and_processors_dictionaries): - """ - Test that the bindings between a puller and a processor are correctly created - """ - pullers = pre_processor_pullers_and_processors_dictionaries[0] - processors = pre_processor_pullers_and_processors_dictionaries[1] - - binding_manager = PreProcessorBindingManager(actors=pullers, - processors=processors) - - assert len(pullers['one_puller'].state.report_filter.filters) == 1 - assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], DispatcherActor) - - binding_manager.process_bindings() - - assert len(pullers['one_puller'].state.report_filter.filters) == 1 - assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], ProcessorActor) - assert pullers['one_puller'].state.report_filter.filters[0][1] == processors['my_processor'] - - -def test_process_bindings_for_pre_processor_raise_exception_with_wrong_binding_types( - pre_processor_binding_manager_with_wrong_binding_types): - """ - Test that an exception is raised with a wrong type for the from actor in a binding - """ - - with pytest.raises(UnsupportedActorTypeException): - pre_processor_binding_manager_with_wrong_binding_types.process_bindings() - - -def test_process_bindings_for_pre_processor_raise_exception_with_no_existing_puller( - pre_processor_binding_manager_with_unexisting_puller): - """ - Test that an exception is raised with a puller that doesn't exist - """ - - with pytest.raises(UnexistingActorException): - pre_processor_binding_manager_with_unexisting_puller.process_bindings() - - -def test_process_bindings_for_pre_processor_raise_exception_with_reused_puller_in_bindings( - pre_processor_binding_manager_with_reused_puller_in_bindings): - """ - Test that an exception is raised when the same puller is used by several processors - """ - - with pytest.raises(TargetActorAlreadyUsed): - pre_processor_binding_manager_with_reused_puller_in_bindings.process_bindings() - - -def test_check_processors_targets_are_unique_raise_exception_with_reused_puller_in_bindings( - pre_processor_binding_manager_with_reused_puller_in_bindings): - """ - Test that an exception is raised when the same puller is used by several processors - """ - with pytest.raises(TargetActorAlreadyUsed): - pre_processor_binding_manager_with_reused_puller_in_bindings.check_processors_targets_are_unique() - - -def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindings( - pre_processor_binding_manager): - """ - Test that a correct without repeated target passes the validation - """ - try: - pre_processor_binding_manager.check_processors_targets_are_unique() - assert True - except TargetActorAlreadyUsed: - assert False - - -def test_check_processor_targets_raise_exception_with_no_existing_puller( - pre_processor_binding_manager_with_unexisting_puller): - """ - Test that an exception is raised with a puller that doesn't exist - """ - pre_processor_binding_manager = pre_processor_binding_manager_with_unexisting_puller - with pytest.raises(UnexistingActorException): - for _, processor in pre_processor_binding_manager.processors.items(): - pre_processor_binding_manager.check_processor_targets(processor=processor) - - -def test_check_processor_targets_raise_exception_with_raise_exception_with_wrong_binding_types( - pre_processor_binding_manager_with_wrong_binding_types): - """ - Test that an exception is raised with a puller that doesn't exist - """ - pre_processor_binding_manager = pre_processor_binding_manager_with_wrong_binding_types - with pytest.raises(UnsupportedActorTypeException): - for _, processor in pre_processor_binding_manager.processors.items(): - pre_processor_binding_manager.check_processor_targets(processor=processor) - - -def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding_manager): - """ - Test that validation of a configuration with existing targets of the correct type - """ - try: - for _, processor in pre_processor_binding_manager.processors.items(): - pre_processor_binding_manager.check_processor_targets(processor=processor) - - assert True - except UnsupportedActorTypeException: - assert False - except UnexistingActorException: - assert False diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 75374bcd..01328bea 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -36,7 +36,7 @@ from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator, \ MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY, PreProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist -from powerapi.processor.pre.k8s.k8s_monitor_actor import K8sMonitorAgentActor +from powerapi.processor.pre.k8s.k8s_monitor import K8sMonitorAgent from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, TIME_INTERVAL_DEFAULT_VALUE, \ TIMEOUT_QUERY_DEFAULT_VALUE from powerapi.processor.pre.libvirt.libvirt_pre_processor_actor import LibvirtPreProcessorActor @@ -536,18 +536,22 @@ def test_generate_k8s_pre_processor_uses_default_values_with_missing_arguments( pre_processor_number += 1 -def check_k8s_monitor_infos(monitor: K8sMonitorAgentActor, associated_processor: K8sPreProcessorActor): +def check_k8s_monitor_infos(monitor: K8sMonitorAgent, associated_processor: K8sPreProcessorActor): """ Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor """ - assert isinstance(monitor, K8sMonitorAgentActor) + assert isinstance(monitor, K8sMonitorAgent) - assert monitor.state.k8s_api_mode == associated_processor.state.k8s_api_mode + assert monitor.concerned_actor_state.k8s_api_mode == associated_processor.state.k8s_api_mode - assert monitor.state.time_interval == associated_processor.state.time_interval + assert monitor.concerned_actor_state.time_interval == associated_processor.state.time_interval - assert monitor.state.timeout_query == associated_processor.state.timeout_query + assert monitor.concerned_actor_state.timeout_query == associated_processor.state.timeout_query + + assert monitor.concerned_actor_state.api_key == associated_processor.state.api_key + + assert monitor.concerned_actor_state.host == associated_processor.state.host def test_generate_k8s_monitor_from_k8s_config(k8s_monitor_config): diff --git a/tests/unit/processor/conftest.py b/tests/unit/processor/conftest.py index 77699bf9..d973891a 100644 --- a/tests/unit/processor/conftest.py +++ b/tests/unit/processor/conftest.py @@ -1,10 +1,9 @@ # Copyright (c) 2023, INRIA # Copyright (c) 2023, University of Lille # All rights reserved. -from typing import Any # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: - +import multiprocessing # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. @@ -27,12 +26,13 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from typing import Any + from unittest.mock import Mock import pytest -from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sMetadataCache +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPodUpdateMetadata, K8sMetadataCacheManager @pytest.fixture(name='pods_list') @@ -151,21 +151,20 @@ def extract_containers_ids(containers_status: list) -> list: @pytest.fixture -def expected_k8s_pod_update_messages(expected_events_list_k8s): +def expected_k8s_pod_update_metadata(expected_events_list_k8s): """ Return a list of K8sPodUpdateMessage by using the provided events list """ - update_messages = [] + update_metadata = [] - for type, namespace, name, containers_id, labels in expected_events_list_k8s: - update_messages.append(K8sPodUpdateMessage(sender_name='test_k8s_monitor_actor', - event=type, - namespace=namespace, - pod=name, - containers_id=containers_id, - labels=labels)) + for event_type, namespace, name, containers_id, labels in expected_events_list_k8s: + update_metadata.append(K8sPodUpdateMetadata(event=event_type, + namespace=namespace, + pod=name, + containers_id=containers_id, + labels=labels)) - return update_messages + return update_metadata class MockedWatch(Mock): @@ -206,22 +205,23 @@ def mocked_watch_initialized_unknown_events(unknown_events_list_k8s): return MockedWatch(unknown_events_list_k8s) -class PipeMetadataCache(K8sMetadataCache): +class PipeMetadataCacheManager(K8sMetadataCacheManager): """ - K8sMetadataCache maintains a cache of pods' metadata + K8sMetadataCacheManager maintains a cache of pods' metadata (namespace, labels and id of associated containers). This metadata cache send itself via a pipe when an update is done """ def __init__(self, name: str, level_logger: int, pipe): - K8sMetadataCache.__init__(self, name=name, level_logger=level_logger) + K8sMetadataCacheManager.__init__(self, name=name, level_logger=level_logger, + process_manager=multiprocessing.Manager()) self.pipe = pipe - def update_cache(self, message: K8sPodUpdateMessage): + def update_cache(self, metadata: K8sPodUpdateMetadata): """ Update the local cache for pods. Send the metadata cache via the pipe """ - K8sMetadataCache.update_cache(self, message=message) + K8sMetadataCacheManager.update_cache(self, metadata=metadata) self.pipe.send(self) diff --git a/tests/unit/processor/pre/k8s/test_k8s_monitor.py b/tests/unit/processor/pre/k8s/test_k8s_monitor.py index 78b6579e..65413756 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_monitor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_monitor.py @@ -1,7 +1,7 @@ # Copyright (c) 2023, INRIA # Copyright (c) 2023, University of Lille # All rights reserved. - +from time import sleep # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -35,10 +35,12 @@ from kubernetes import client -from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.pre.k8s.k8s_monitor_actor import local_config, MANUAL_CONFIG_MODE, \ - K8sMonitorAgentActor -from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe +from powerapi.processor.pre.k8s.k8s_monitor import local_config, MANUAL_CONFIG_MODE, \ + K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPodUpdateMetadata, K8sPreProcessorState, ADDED_EVENT, \ + MODIFIED_EVENT +from powerapi.report import HWPCReport +from tests.utils.actor.dummy_actor import DummyActor LISTENER_AGENT_NAME = 'test_k8s_processor_listener_agent' @@ -59,9 +61,9 @@ def test_load_local_config(): assert ret.items != [] -class TestK8sMonitor(AbstractTestActor): +class TestK8sMonitor: """ - Class for testing a monitor actor + Class for testing a monitor """ @pytest.fixture @@ -69,53 +71,97 @@ def report_to_be_sent(self): """ This fixture must return the report class for testing """ - return K8sPodUpdateMessage + return K8sPodUpdateMetadata @pytest.fixture - def actor(self, started_fake_target_actor, mocked_watch_initialized, pods_list): + def monitor_agent(self, mocked_watch_initialized, pods_list): + """ + Return a monitor agent that uses the provided mocked watch + """ with patch('kubernetes.client.CoreV1Api', return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): with patch('kubernetes.config.load_kube_config', return_value=Mock()): with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): - yield K8sMonitorAgentActor(name='test_k8s_monitor_actor', - listener_agent=started_fake_target_actor, - k8s_api_mode=MANUAL_CONFIG_MODE) - - def test_streaming_query(self, started_actor, pods_list, expected_events_list_k8s, mocked_watch_initialized, + monitor_agent = K8sMonitorAgent(name='test_k8s_monitor', + concerned_actor_state=K8sPreProcessorState( + actor=DummyActor(name='test_k8s_monitor_actor', + pipe=None, message_type=HWPCReport), + target_actors=[], + target_actors_names=[], + k8s_api_mode=MANUAL_CONFIG_MODE, + time_interval=10, + timeout_query=10, + api_key='', + host='' + ) + ) + yield monitor_agent + + def test_streaming_query(self, monitor_agent, pods_list, expected_events_list_k8s, mocked_watch_initialized, shutdown_system): """ Test that k8s_streaming_query is able to retrieve events related to pods """ - result = started_actor.k8s_streaming_query() + result = monitor_agent.k8s_streaming_query() assert result == expected_events_list_k8s def test_unknown_events_streaming_query(self, pods_list, mocked_watch_initialized_unknown_events, - started_actor, shutdown_system): + monitor_agent, shutdown_system): """ Test that unknown events are ignored by k8s_streaming_query """ - result = started_actor.k8s_streaming_query() + result = monitor_agent.k8s_streaming_query() assert result == [] - def test_monitor_send_message_k8s_pod_update_message_when_events_are_available(self, started_actor, - expected_k8s_pod_update_messages, - dummy_pipe_out, shutdown_system): + def test_monitor_agent_update_metadata_cache_when_events_are_available(self, monitor_agent, + expected_k8s_pod_update_metadata, + shutdown_system): """ - Test that the monitor sends to the target an update message when events are available + Test that the monitor updates metadata cache when events are available """ - messages_found = 0 + expected_pods_labels_size = 0 + expected_containers_ids_size = 0 + for current_metadata in expected_k8s_pod_update_metadata: + if current_metadata.event in [ADDED_EVENT, MODIFIED_EVENT]: + expected_pods_labels_size += 1 + expected_containers_ids_size += len(current_metadata.containers_id) + + monitor_agent.start() + + sleep(1) + + monitor_agent.stop_monitoring.set() + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) == \ + expected_pods_labels_size # There is a event if each type ADDED, DELETED and MODIFIED + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) == \ + expected_pods_labels_size + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) == \ + expected_containers_ids_size + + assert monitor_agent.stop_monitoring.is_set() + + def test_stop_monitor_agent_works(self, monitor_agent): + """ + Test that monitor agent is correctly stopped when the flag related to the monitoring is changed + """ + + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) == 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) == 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) == 0 + assert not monitor_agent.stop_monitoring.is_set() - for _ in range(len(expected_k8s_pod_update_messages)): - result = recv_from_pipe(dummy_pipe_out, 2) - got_message = result[1] - assert isinstance(got_message, K8sPodUpdateMessage) + monitor_agent.start() - for expected_message in expected_k8s_pod_update_messages: + sleep(1) - if got_message == expected_message: - messages_found += 1 - break + monitor_agent.stop_monitoring.set() - assert messages_found == len(expected_k8s_pod_update_messages) + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_labels) > 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.pod_containers) > 0 + assert len(monitor_agent.concerned_actor_state.metadata_cache_manager.containers_pod) > 0 + assert monitor_agent.stop_monitoring.is_set() diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py index c3fad7bc..b1278ba4 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -31,23 +31,46 @@ import logging from copy import deepcopy +from time import sleep from unittest.mock import patch, Mock import pytest -from powerapi.message import K8sPodUpdateMessage -from powerapi.processor.pre.k8s.k8s_monitor_actor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT -from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor +from powerapi.processor.pre.k8s.k8s_monitor import MANUAL_CONFIG_MODE, ADDED_EVENT, MODIFIED_EVENT, DELETED_EVENT, \ + K8sMonitorAgent +from powerapi.processor.pre.k8s.k8s_pre_processor_actor import K8sPreProcessorActor, K8sPodUpdateMetadata from powerapi.processor.pre.k8s.k8s_pre_processor_handlers import clean_up_container_id, POD_NAMESPACE_METADATA_KEY, \ POD_NAME_METADATA_KEY from powerapi.report import HWPCReport from tests.unit.actor.abstract_test_actor import AbstractTestActor, recv_from_pipe -from tests.unit.processor.conftest import PipeMetadataCache +from tests.unit.processor.conftest import MockedWatch, FakePod, FakeMetadata, FakeStatus, FakeContainerStatus from tests.utils.report.hwpc import extract_rapl_reports_with_2_sockets DISPATCHER_NAME = 'test_k8s_processor_dispatcher' +def get_metadata_from_event(basic_event: dict): + """ + Create a K8sPodUpdateMetadata from a dict containing the event info + :param dict basic_event : The event information + """ + containers_id = [] + + for current_fake_container_status in basic_event['object'].status.container_statuses: + index_id = current_fake_container_status.container_id.index('://') + current_container_id = current_fake_container_status.container_id[index_id + + len('://'): + len(current_fake_container_status.container_id + )] + containers_id.append(current_container_id) + + return K8sPodUpdateMetadata(event=basic_event['type'], + namespace=basic_event['object'].metadata.namespace, + pod=basic_event['object'].metadata.name, + containers_id=containers_id, + labels=basic_event['object'].metadata.labels) + + def test_clean_up_id_docker(): """ Test that the cleanup of the docker id works correctly @@ -75,207 +98,260 @@ class TestK8sProcessor(AbstractTestActor): """ @pytest.fixture - def multiprocess_metadata_cache_empty(self, dummy_pipe_in): + def report_to_be_sent(self): """ - Create a metadata cache that send the object once it is modified via a pipe + This fixture must return the report class for testing """ - return PipeMetadataCache(name='test_k8s_process', level_logger=logging.DEBUG, pipe=dummy_pipe_in) + return HWPCReport @pytest.fixture - def update_metadata_cache_message_added_event(self): + def basic_added_event_k8s(self): """ - Create a update message + Return a basic ADDED event and its related information """ - return K8sPodUpdateMessage(sender_name='test_k8s_processor_added', - event=ADDED_EVENT, - namespace='test_k8s_processor_namespace', - pod='test_k8s_processor_pod', - containers_id=[ - 'test_cid_1_added', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_2_added', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_3_added', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_4_added'], - labels={'l1': 'v1', 'l2': 'v2', 'l3': 'v3'}) - - @pytest.fixture - def update_metadata_cache_message_modified_event(self): - """ - Create a update message - """ - return K8sPodUpdateMessage(sender_name='test_k8s_processor_modified', - event=MODIFIED_EVENT, - namespace='test_k8s_processor_namespace', - pod='test_k8s_processor_pod', - containers_id=[ - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_1_modified', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_2_modified', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_3_modified', - '/kubepods/test_qos/pod_test_k8s_processor_pod_added/test_cid_4_modified'], - labels={'l1': 'v1_modified', 'l2_modified': 'v2_modified', - 'l3_modified': 'v3_modified'}) + return { + 'type': ADDED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels={'l1': 'v1', 'l2': 'v2', 'l3': 'v3', 'l4': 'v4', 'l5': 'v5'}), + status=FakeStatus(container_statuses=[FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_1_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_2_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_3_added'), + FakeContainerStatus( + container_id='/kpods/t_q/pod_test_k8s_processor_pod_added://test_cid_4_added')]) + ) + } @pytest.fixture - def update_metadata_cache_message_deleted_event(self): + def basic_modified_event_k8s(self): """ - Create a update message with DELETED as event + Return a basic MODIFIED event and its related information """ - return K8sPodUpdateMessage(sender_name='test_k8s_processor_deleted', - event=DELETED_EVENT, - namespace='test_k8s_processor_namespace', - pod='test_k8s_processor_pod') + return { + 'type': MODIFIED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels={'l1_m': 'v1', 'l2_m': 'v2', 'l3_m': 'v3', 'l4_m': 'v4', 'l5_m': 'v5'}), + status=FakeStatus(container_statuses=[FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_1_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_2_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_3_modified'), + FakeContainerStatus( + container_id='/kp/t_q/pod_test_k8s_processor_pod_added://test_cid_4_modified')]) + ) + } @pytest.fixture - def update_metadata_cache_message_unknown_event(self): + def basic_deleted_event_k8s(self): """ - Create a update message + Return a basic DELETED event and its related information """ - return K8sPodUpdateMessage(sender_name='test_k8s_processor_unknown', - event='Unknown Event', - namespace='test_k8s_processor_namespace', - pod='test_k8s_processor_pod') + return { + 'type': DELETED_EVENT, + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels=[]), + status=None + ) + } @pytest.fixture - def init_multiprocess_metadata_cache_with_data(self, started_actor, update_metadata_cache_message_added_event, - dummy_pipe_out): + def basic_unknown_event_k8s(self): """ - Initialize the metadata cache of the actor + Return a basic DELETED event and its related information """ - started_actor.send_data(update_metadata_cache_message_added_event) - _ = recv_from_pipe(dummy_pipe_out, 2) - return started_actor + return { + 'type': 'Unknown Event', + 'object': FakePod( + metadata=FakeMetadata(name='test_k8s_processor_pod', namespace='test_k8s_processor_namespace', + labels=[]), + status=None + ) + } @pytest.fixture() - def hwpc_report(self, update_metadata_cache_message_added_event): + def hwpc_report(self, basic_added_event_k8s): """ Return a HWPC Report """ json_input = extract_rapl_reports_with_2_sockets(1)[0] report = HWPCReport.from_json(json_input) - report.target = update_metadata_cache_message_added_event.containers_id[0] + report.target = basic_added_event_k8s['object'].status.container_statuses[0].container_id return report @pytest.fixture() - def hwpc_report_with_metadata(self, hwpc_report, update_metadata_cache_message_added_event): + def hwpc_report_with_metadata(self, hwpc_report, basic_added_event_k8s): """ Return a HWPC report with metadata """ + update_metadata_cache_added_event = get_metadata_from_event(basic_event=basic_added_event_k8s) hwpc_report_with_metadata = deepcopy(hwpc_report) hwpc_report_with_metadata.metadata[POD_NAMESPACE_METADATA_KEY] = \ - update_metadata_cache_message_added_event.namespace - hwpc_report_with_metadata.metadata[POD_NAME_METADATA_KEY] = update_metadata_cache_message_added_event.pod + update_metadata_cache_added_event.namespace + hwpc_report_with_metadata.metadata[POD_NAME_METADATA_KEY] = update_metadata_cache_added_event.pod - for label_name, label_value in update_metadata_cache_message_added_event.labels.items(): + for label_name, label_value in update_metadata_cache_added_event.labels.items(): hwpc_report_with_metadata.metadata[f"label_{label_name}"] = label_value return hwpc_report_with_metadata @pytest.fixture - def report_to_be_sent(self): + def actor(self, started_fake_target_actor, pods_list, mocked_watch_initialized): + return K8sPreProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, + target_actors=[started_fake_target_actor], + level_logger=logging.DEBUG) + + @pytest.fixture + def mocked_monitor_added_event(self, actor, basic_added_event_k8s, pods_list): """ - This fixture must return the report class for testing + Return a mocked monitor that produces an added event """ - return HWPCReport + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', + return_value=MockedWatch(events=[basic_added_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + @pytest.fixture + def mocked_monitor_modified_event(self, actor, basic_modified_event_k8s, pods_list): + """ + Return a mocked monitor that produces a modified event + """ + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', + return_value=MockedWatch(events=[basic_modified_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) @pytest.fixture - def actor(self, started_fake_target_actor, pods_list, mocked_watch_initialized, - multiprocess_metadata_cache_empty): + def mocked_monitor_deleted_event(self, actor, basic_deleted_event_k8s, pods_list): + """ + Return a mocked monitor that produces a deleted event + """ with patch('kubernetes.client.CoreV1Api', return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): with patch('kubernetes.config.load_kube_config', return_value=Mock()): - with patch('kubernetes.watch.Watch', return_value=mocked_watch_initialized): - with patch('powerapi.processor.pre.k8s.k8s_pre_processor_actor.K8sMetadataCache', - return_value=multiprocess_metadata_cache_empty): - return K8sPreProcessorActor(name='test_k8s_processor_actor', ks8_api_mode=MANUAL_CONFIG_MODE, - target_actors=[started_fake_target_actor], - level_logger=logging.DEBUG) + with patch('kubernetes.watch.Watch', + return_value=MockedWatch(events=[basic_deleted_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) - def test_update_metadata_cache_with_added_event(self, started_actor, update_metadata_cache_message_added_event, - dummy_pipe_out, shutdown_system): + @pytest.fixture + def mocked_monitor_unknown_event(self, actor, basic_added_event_k8s, basic_unknown_event_k8s, pods_list): + """ + Return a mocked monitor that produces an unknown event + """ + with patch('kubernetes.client.CoreV1Api', + return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): + with patch('kubernetes.config.load_kube_config', return_value=Mock()): + with patch('kubernetes.watch.Watch', + return_value=MockedWatch(events=[basic_unknown_event_k8s, basic_added_event_k8s])): + yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', + concerned_actor_state=actor.state) + + def test_update_metadata_cache_with_added_event(self, mocked_monitor_added_event, started_actor, + basic_added_event_k8s, shutdown_system): """ Test that metadata_cache is correctly updated when a reception of an added event """ - update_message = update_metadata_cache_message_added_event - started_actor.send_data(update_message) + metadata_for_update = get_metadata_from_event(basic_event=basic_added_event_k8s) + mocked_monitor_added_event.start() + sleep(1) - result = recv_from_pipe(dummy_pipe_out, 2) + result = started_actor.state.metadata_cache_manager + + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id - assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message.labels - assert result.pod_containers[(update_message.namespace, update_message.pod)] == update_message.containers_id + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) - for container_id in update_message.containers_id: - assert result.containers_pod[container_id] == (update_message.namespace, update_message.pod) + mocked_monitor_added_event.stop_monitoring.set() - def test_update_metadata_cache_with_modified_event(self, init_multiprocess_metadata_cache_with_data, - update_metadata_cache_message_modified_event, dummy_pipe_out, - shutdown_system): + def test_update_metadata_cache_with_modified_event(self, mocked_monitor_modified_event, started_actor, + basic_modified_event_k8s, shutdown_system): """ Test that metadata_cache is correctly updated when a reception of a modified event """ - started_actor = init_multiprocess_metadata_cache_with_data - update_message = update_metadata_cache_message_modified_event - started_actor.send_data(update_message) + metadata_for_update = get_metadata_from_event(basic_event=basic_modified_event_k8s) + mocked_monitor_modified_event.start() + sleep(2) - result = recv_from_pipe(dummy_pipe_out, 2) + result = started_actor.state.metadata_cache_manager - assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message.labels - assert result.pod_containers[(update_message.namespace, update_message.pod)] == update_message.containers_id + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id - for container_id in update_message.containers_id: - assert result.containers_pod[container_id] == (update_message.namespace, update_message.pod) + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) - def test_update_metadata_cache_with_deleted_event(self, init_multiprocess_metadata_cache_with_data, - update_metadata_cache_message_deleted_event, dummy_pipe_out, - shutdown_system): + mocked_monitor_modified_event.stop_monitoring.set() + + def test_update_metadata_cache_with_deleted_event(self, mocked_monitor_deleted_event, started_actor, + basic_deleted_event_k8s, shutdown_system): """ Test that metadata_cache is correctly updated when a reception of a deleted event """ - started_actor = init_multiprocess_metadata_cache_with_data - - update_message = update_metadata_cache_message_deleted_event - - started_actor.send_data(update_message) + mocked_monitor_deleted_event.start() + sleep(0.5) - result = recv_from_pipe(dummy_pipe_out, 2) + result = started_actor.state.metadata_cache_manager assert len(result.pod_labels) == 0 assert len(result.pod_containers) == 0 assert len(result.containers_pod) == 0 - def test_update_metadata_cache_with_unknown_event_does_not_modify_it(self, - init_multiprocess_metadata_cache_with_data, - update_metadata_cache_message_unknown_event, - dummy_pipe_out, - update_metadata_cache_message_added_event, + mocked_monitor_deleted_event.stop_monitoring.set() + + def test_update_metadata_cache_with_unknown_event_does_not_modify_it(self, mocked_monitor_unknown_event, + started_actor, + basic_added_event_k8s, shutdown_system): """ - Test that metadata_cache is not updated when a reception of a unknown event + Test that metadata_cache is not updated when a reception of an unknown event """ - started_actor = init_multiprocess_metadata_cache_with_data + metadata_for_update = get_metadata_from_event(basic_event=basic_added_event_k8s) + mocked_monitor_unknown_event.start() - update_message = update_metadata_cache_message_unknown_event - update_message_added = update_metadata_cache_message_added_event - started_actor.send_data(update_message) + sleep(1) - result = recv_from_pipe(dummy_pipe_out, 2) + result = started_actor.state.metadata_cache_manager - assert result.pod_labels[(update_message.namespace, update_message.pod)] == update_message_added.labels - assert result.pod_containers[(update_message.namespace, update_message.pod)] == \ - update_message_added.containers_id + assert result.pod_labels[(metadata_for_update.namespace, metadata_for_update.pod)] == metadata_for_update.labels + assert result.pod_containers[(metadata_for_update.namespace, metadata_for_update.pod)] == \ + metadata_for_update.containers_id - for container_id in update_message.containers_id: - assert result.containers_pod[container_id] == (update_message_added.namespace, update_message_added.pod) + for container_id in metadata_for_update.containers_id: + assert result.containers_pod[container_id] == (metadata_for_update.namespace, metadata_for_update.pod) - def test_add_metadata_to_hwpc_report(self, - init_multiprocess_metadata_cache_with_data, + mocked_monitor_unknown_event.stop_monitoring.set() + + def test_add_metadata_to_hwpc_report(self, mocked_monitor_added_event, + started_actor, hwpc_report, hwpc_report_with_metadata, - dummy_pipe_out, shutdown_system): - """ - Test that a HWPC report is modified with the correct metadata + dummy_pipe_out, + shutdown_system): """ - started_actor = init_multiprocess_metadata_cache_with_data + Test that a HWPC report is modified with the correct metadata + """ + mocked_monitor_added_event.start() + + sleep(1) started_actor.send_data(hwpc_report) @@ -283,17 +359,20 @@ def test_add_metadata_to_hwpc_report(self, assert result[1] == hwpc_report_with_metadata + mocked_monitor_added_event.stop_monitoring.set() + def test_add_metadata_to_hwpc_report_does_not_modify_report_with_unknown_container_id(self, + mocked_monitor_added_event, started_actor, hwpc_report, dummy_pipe_out, shutdown_system): """ - Test that a HWPC report is not modified with an unknown container id - """ + Test that a HWPC report is not modified with an unknown container id + """ started_actor.send_data(hwpc_report) - result = recv_from_pipe(dummy_pipe_out, 2) + result = recv_from_pipe(dummy_pipe_out, 4) assert result[1] == hwpc_report From 0623dc7d27cebe0d05e3ea74ffe7c2608f2a2f90 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 31 Oct 2023 09:39:10 +0100 Subject: [PATCH 20/35] refactor: Readd BindingManager as it is required to create binding between processors and pullers/pushers and its related tests, Improve K8sMonitorAgent exception management --- powerapi/cli/binding_manager.py | 232 ++++++++++++++++++++++ powerapi/processor/pre/k8s/k8s_monitor.py | 8 +- tests/unit/cli/conftest.py | 46 +++++ tests/unit/cli/test_binding_manager.py | 172 ++++++++++++++++ 4 files changed, 456 insertions(+), 2 deletions(-) create mode 100644 powerapi/cli/binding_manager.py create mode 100644 tests/unit/cli/test_binding_manager.py diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py new file mode 100644 index 00000000..4e00fe80 --- /dev/null +++ b/powerapi/cli/binding_manager.py @@ -0,0 +1,232 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed +from powerapi.processor.processor_actor import ProcessorActor +from powerapi.puller import PullerActor +from powerapi.pusher import PusherActor + + +class BindingManager: + """ + Class for management the binding between actors during their creation process + """ + + def __init__(self, actors: dict = {}): + """ + :param dict actors: Dictionary of actors to create the bindings. The name of the actor is the key + """ + if not actors: + self.actors = {} + else: + self.actors = actors + + def process_bindings(self): + """ + Define bindings between self.actors according to the processors' targets. + """ + raise NotImplementedError() + + +class ProcessorBindingManager(BindingManager): + """ + Class for management of bindings between processor actors and others actors + """ + + def __init__(self, actors: dict, processors: dict): + """ + The ProcessorBindingManager defines bindings between actors and processors + :param dict actors: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + + BindingManager.__init__(self, actors=actors) + if not processors: + self.processors = {} + else: + self.processors = processors + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + """ + for target_actor_name in processor.state.target_actors_names: + if target_actor_name not in self.actors: + raise UnexistingActorException(actor=target_actor_name) + + def check_processors_targets_are_unique(self): + """ + Check that processors targets are unique, i.e., the same target is not related to + two different processors + """ + used_targets = [] + for _, processor in self.processors.items(): + for target_actor_name in processor.state.target_actors_names: + if target_actor_name in used_targets: + raise TargetActorAlreadyUsed(target_actor=target_actor_name) + else: + used_targets.append(target_actor_name) + + +class PreProcessorBindingManager(ProcessorBindingManager): + """ + Class for management the binding between pullers and pre-processor actors + """ + + def __init__(self, pullers: dict, processors: dict): + """ + The PreProcessorBindingManager defines bindings between pullers and processors: puller->processor->dispatcher + :param dict pullers: Dictionary of actors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + + ProcessorBindingManager.__init__(self, actors=pullers, processors=processors) + + def process_bindings(self): + """ + Define bindings between self.actors according to the pre-processors' targets. + + """ + + # Check that processors targets are unique + self.check_processors_targets_are_unique() + + # For each processor, we get targets and create the binding: + # puller->processor->dispatcher + for _, processor in self.processors.items(): + + self.check_processor_targets(processor=processor) + + for target_actor_name in processor.state.target_actors_names: + + # The processor has to be between the puller and the dispatcher + # The dispatcher becomes a target of the processor + + puller_actor = self.actors[target_actor_name] + + # The dispatcher defines the relationship between the Formula and + # Puller + number_of_filters = len(puller_actor.state.report_filter.filters) + + for index in range(number_of_filters): + # The filters define the relationship with the dispatcher + # The relationship has to be updated + current_filter = list(puller_actor.state.report_filter.filters[index]) + current_filter_dispatcher = current_filter[1] + processor.add_target_actor(actor=current_filter_dispatcher) + current_filter[1] = processor + puller_actor.state.report_filter.filters[index] = tuple(current_filter) + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PullerActor instance. + If it is not the case, it raises UnsupportedActorTypeException + """ + ProcessorBindingManager.check_processor_targets(self, processor=processor) + + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] + + if not isinstance(actor, PullerActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) + + +class PostProcessorBindingManager(ProcessorBindingManager): + """ + Class for management the binding between post-processor and pusher actors + """ + + def __init__(self, pushers: dict, processors: dict, pullers: dict): + """ + The PostProcessorBindingManager defines bindings between processors and pushers: formula->processor->pushers + :param dict pushers: Dictionary of PusherActors with structure {:actor1,:actor2...} + :param dict processors: Dictionary of processors with structure {:processor1, + :processor2...} + """ + ProcessorBindingManager.__init__(self, actors=pushers, processors=processors) + self.pullers = pullers + + def process_bindings(self): + """ + Define bindings between self.actors according to the post-processors' targets. + + """ + + # For each processor, we get targets and create the binding: + # formula->processor->pusher + for _, processor in self.processors.items(): + + self.check_processor_targets(processor=processor) + + for target_actor_name in processor.state.target_actors_names: + + # The processor has to be between the formula and the pusher + # The pusher becomes a target of the processor + + pusher_actor = self.actors[target_actor_name] + + processor.add_target_actor(actor=pusher_actor) + + # We look for the pusher on each dispatcher in order to replace it by + # the processor + for _, puller in self.pullers: + + for current_filter in puller.state.report_filter.filters: + dispatcher = current_filter[1] + + number_of_pushers = len(dispatcher.pusher) + pusher_updated = False + + for index in range(number_of_pushers): + if dispatcher.pusher[index] == pusher_actor: + dispatcher.pusher[index] = processor + pusher_updated = True + break + + if pusher_updated: + dispatcher.update_state_formula_factory() + + def check_processor_targets(self, processor: ProcessorActor): + """ + Check that targets of a processor exist in the dictionary of targets. + If it is not the case, it raises a UnexistingActorException + It also checks that the actor is a PusherActor instance. + If it is not the case, it raises UnsupportedActorTypeException + """ + ProcessorBindingManager.check_processor_targets(self, processor=processor) + + for target_actor_name in processor.state.target_actors_names: + actor = self.actors[target_actor_name] + + if not isinstance(actor, PusherActor): + raise UnsupportedActorTypeException(actor_type=type(actor).__name__) diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py index 01deff7b..39b0f348 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -199,10 +199,14 @@ def query_k8s(self): ) self.stop_monitoring.wait(timeout=self.concerned_actor_state.time_interval) + + except BrokenPipeError: + # This error can happen when stopping the monitor process + return except Exception as ex: self.logger.warning("Failed streaming query %s", ex) - - self.manager.shutdown() + finally: + self.manager.shutdown() def k8s_streaming_query(self) -> list: """ diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 786fb058..1db8670e 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -31,6 +31,7 @@ import pytest import tests.utils.cli as test_files_module +from powerapi.cli.binding_manager import PreProcessorBindingManager from powerapi.cli.generator import PullerGenerator, PusherGenerator, ProcessorGenerator, COMPONENT_TYPE_KEY, \ LISTENER_ACTOR_KEY, MONITOR_NAME_SUFFIX, PreProcessorGenerator from powerapi.dispatcher import DispatcherActor, RouteTable @@ -714,3 +715,48 @@ def dispatcher_actor_in_dictionary(): route_table=route_table) return {dispatcher.name: dispatcher} + + +@pytest.fixture +def pre_processor_binding_manager(pre_processor_pullers_and_processors_dictionaries): + """ + Return a ProcessorBindingManager with a libvirt/K8s Processor + """ + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] + + return PreProcessorBindingManager(pullers=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_binding_configuration): + """ + Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_wrong_binding_configuration) + + return PreProcessorBindingManager(pullers=pushers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unexisting_puller_configuration): + """ + Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_unexisting_puller_configuration) + + return PreProcessorBindingManager(pullers=pullers, processors=processors) + + +@pytest.fixture +def pre_processor_binding_manager_with_reused_puller_in_bindings( + pre_processor_with_reused_puller_in_bindings_configuration): + """ + Return a PreProcessorBindingManager with a puller used by two different pre-processors + """ + pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + configuration=pre_processor_with_reused_puller_in_bindings_configuration) + + return PreProcessorBindingManager(pullers=pullers, processors=processors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py new file mode 100644 index 00000000..02bc59e9 --- /dev/null +++ b/tests/unit/cli/test_binding_manager.py @@ -0,0 +1,172 @@ +# Copyright (c) 2023, INRIA +# Copyright (c) 2023, University of Lille +# All rights reserved. + + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. + +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import copy + +import pytest + +from powerapi.cli.binding_manager import PreProcessorBindingManager +from powerapi.dispatcher import DispatcherActor +from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, \ + TargetActorAlreadyUsed +from powerapi.processor.processor_actor import ProcessorActor + + +def test_create_pre_processor_binding_manager_with_actors(pre_processor_pullers_and_processors_dictionaries): + """ + Test that a PreProcessorBindingManager is correctly created when an actor and a processor dictionary are provided + """ + expected_actors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[0]) + expected_processors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[1]) + + binding_manager = PreProcessorBindingManager(pullers=pre_processor_pullers_and_processors_dictionaries[0], + processors=pre_processor_pullers_and_processors_dictionaries[1]) + + assert binding_manager.actors == expected_actors_dictionary + assert binding_manager.processors == expected_processors_dictionary + + +def test_create_processor_binding_manager_without_actors(): + """ + Test that a ProcessorBindingManager is correctly created without a dictionary + """ + binding_manager = PreProcessorBindingManager(pullers=None, processors=None) + + assert len(binding_manager.actors) == 0 + assert len(binding_manager.processors) == 0 + + +def test_process_bindings_for_pre_processor(pre_processor_complete_configuration, + pre_processor_pullers_and_processors_dictionaries): + """ + Test that the bindings between a puller and a processor are correctly created + """ + pullers = pre_processor_pullers_and_processors_dictionaries[0] + processors = pre_processor_pullers_and_processors_dictionaries[1] + + binding_manager = PreProcessorBindingManager(pullers=pullers, + processors=processors) + + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], DispatcherActor) + + binding_manager.process_bindings() + + assert len(pullers['one_puller'].state.report_filter.filters) == 1 + assert isinstance(pullers['one_puller'].state.report_filter.filters[0][1], ProcessorActor) + assert pullers['one_puller'].state.report_filter.filters[0][1] == processors['my_processor'] + + +def test_process_bindings_for_pre_processor_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): + """ + Test that an exception is raised with a wrong type for the from actor in a binding + """ + + with pytest.raises(UnsupportedActorTypeException): + pre_processor_binding_manager_with_wrong_binding_types.process_bindings() + + +def test_process_bindings_for_pre_processor_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): + """ + Test that an exception is raised with a puller that doesn't exist + """ + + with pytest.raises(UnexistingActorException): + pre_processor_binding_manager_with_unexisting_puller.process_bindings() + + +def test_process_bindings_for_pre_processor_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): + """ + Test that an exception is raised when the same puller is used by several processors + """ + + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.process_bindings() + + +def test_check_processors_targets_are_unique_raise_exception_with_reused_puller_in_bindings( + pre_processor_binding_manager_with_reused_puller_in_bindings): + """ + Test that an exception is raised when the same puller is used by several processors + """ + with pytest.raises(TargetActorAlreadyUsed): + pre_processor_binding_manager_with_reused_puller_in_bindings.check_processors_targets_are_unique() + + +def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindings( + pre_processor_binding_manager): + """ + Test that a correct without repeated target passes the validation + """ + try: + pre_processor_binding_manager.check_processors_targets_are_unique() + assert True + except TargetActorAlreadyUsed: + assert False + + +def test_check_processor_targets_raise_exception_with_no_existing_puller( + pre_processor_binding_manager_with_unexisting_puller): + """ + Test that an exception is raised with a puller that doesn't exist + """ + pre_processor_binding_manager = pre_processor_binding_manager_with_unexisting_puller + with pytest.raises(UnexistingActorException): + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + + +def test_check_processor_targets_raise_exception_with_raise_exception_with_wrong_binding_types( + pre_processor_binding_manager_with_wrong_binding_types): + """ + Test that an exception is raised with a puller that doesn't exist + """ + pre_processor_binding_manager = pre_processor_binding_manager_with_wrong_binding_types + with pytest.raises(UnsupportedActorTypeException): + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + + +def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding_manager): + """ + Test that validation of a configuration with existing targets of the correct type + """ + try: + for _, processor in pre_processor_binding_manager.processors.items(): + pre_processor_binding_manager.check_processor_targets(processor=processor) + + assert True + except UnsupportedActorTypeException: + assert False + except UnexistingActorException: + assert False From 43487229a9a44e8aa2d49b992859a5d9bd111670 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 31 Oct 2023 10:36:10 +0100 Subject: [PATCH 21/35] refactor: Improve logger manangement in processors --- powerapi/cli/generator.py | 4 ++++ powerapi/processor/pre/k8s/k8s_monitor.py | 2 +- tests/unit/cli/test_generator.py | 4 ++-- tests/utils/cli/k8s_pre_processor_configuration.json | 1 + tests/utils/cli/libvirt_pre_processor_configuration.json | 1 + tests/utils/cli/several_k8s_pre_processors_configuration.json | 1 + ...s_pre_processors_without_some_arguments_configuration.json | 1 + .../cli/several_libvirt_pre_processors_configuration.json | 1 + 8 files changed, 12 insertions(+), 3 deletions(-) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index d0a87c4a..f6a7818b 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -406,6 +406,7 @@ def _gen_actor(self, component_config: dict, main_config: dict, actor_name: str) raise PowerAPIException(msg) else: component_config[ACTOR_NAME_KEY] = actor_name + component_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] return self.processor_factory[processor_actor_type](component_config) @@ -438,6 +439,9 @@ def _get_default_processor_factories(self) -> dict: else processor_config[API_KEY_KEY], host=None if HOST_KEY not in processor_config else processor_config[HOST_KEY], + level_logger=logging.DEBUG if + processor_config[GENERAL_CONF_VERBOSE_KEY] else + logging.INFO, target_actors_names=[processor_config[PULLER_NAME_KEY]] ) } diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py index 39b0f348..367fb5cb 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -201,7 +201,7 @@ def query_k8s(self): self.stop_monitoring.wait(timeout=self.concerned_actor_state.time_interval) except BrokenPipeError: - # This error can happen when stopping the monitor process + # This error can happen when stopping the monitor process return except Exception as ex: self.logger.warning("Failed streaming query %s", ex) diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 01328bea..9b2ebc82 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -421,7 +421,7 @@ def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config processors = generator.generate(libvirt_pre_processor_config) - assert len(processors) == len(libvirt_pre_processor_config) + assert len(processors) == len(libvirt_pre_processor_config['pre-processor']) assert 'my_processor' in processors processor = processors['my_processor'] @@ -483,7 +483,7 @@ def test_generate_pre_processor_from_k8s_config(k8s_pre_processor_config): processors = generator.generate(k8s_pre_processor_config) - assert len(processors) == len(k8s_pre_processor_config) + assert len(processors) == len(k8s_pre_processor_config['pre-processor']) assert processor_name in processors processor = processors[processor_name] diff --git a/tests/utils/cli/k8s_pre_processor_configuration.json b/tests/utils/cli/k8s_pre_processor_configuration.json index f82d1732..0f14abf2 100644 --- a/tests/utils/cli/k8s_pre_processor_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_configuration.json @@ -1,4 +1,5 @@ { + "verbose": true, "pre-processor": { "my_processor": { "type": "k8s", diff --git a/tests/utils/cli/libvirt_pre_processor_configuration.json b/tests/utils/cli/libvirt_pre_processor_configuration.json index 903d3d16..362fbba9 100644 --- a/tests/utils/cli/libvirt_pre_processor_configuration.json +++ b/tests/utils/cli/libvirt_pre_processor_configuration.json @@ -1,4 +1,5 @@ { + "verbose": true, "pre-processor": { "my_processor": { "type": "libvirt", diff --git a/tests/utils/cli/several_k8s_pre_processors_configuration.json b/tests/utils/cli/several_k8s_pre_processors_configuration.json index 2a7cba50..e94418ca 100644 --- a/tests/utils/cli/several_k8s_pre_processors_configuration.json +++ b/tests/utils/cli/several_k8s_pre_processors_configuration.json @@ -1,4 +1,5 @@ { + "verbose": true, "pre-processor": { "my_processor_1": { "type": "k8s", diff --git a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json index 5e51ebad..1f068c53 100644 --- a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json +++ b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json @@ -1,4 +1,5 @@ { + "verbose": true, "pre-processor": { "my_processor_1": { "type": "k8s", diff --git a/tests/utils/cli/several_libvirt_pre_processors_configuration.json b/tests/utils/cli/several_libvirt_pre_processors_configuration.json index 4fa686f1..52bf2a3d 100644 --- a/tests/utils/cli/several_libvirt_pre_processors_configuration.json +++ b/tests/utils/cli/several_libvirt_pre_processors_configuration.json @@ -1,4 +1,5 @@ { + "verbose": true, "pre-processor": { "my_processor_1": { "type": "libvirt", From a068652b2c3827878eac4b565d1a2c0bfaa7c033 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 31 Oct 2023 11:44:16 +0100 Subject: [PATCH 22/35] fix: Correct issue related to parametres names in K8s configuration --- powerapi/cli/generator.py | 8 ++--- tests/unit/cli/test_generator.py | 10 +++--- .../cli/k8s_pre_processor_configuration.json | 6 ++-- ...ith_non_existing_puller_configuration.json | 6 ++-- ...used_puller_in_bindings_configuration.json | 12 +++---- ...processor_wrong_binding_configuration.json | 6 ++-- ...eral_k8s_pre_processors_configuration.json | 36 +++++++++---------- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index f6a7818b..ae50edf0 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -62,12 +62,12 @@ ACTOR_NAME_KEY = 'actor_name' TARGET_ACTORS_KEY = 'target_actors' REGEXP_KEY = 'regexp' -K8S_API_MODE_KEY = 'k8s_api_mode' -TIME_INTERVAL_KEY = 'time_interval' -TIMEOUT_QUERY_KEY = 'timeout_query' +K8S_API_MODE_KEY = 'k8s-api-mode' +TIME_INTERVAL_KEY = 'time-interval' +TIMEOUT_QUERY_KEY = 'timeout-query' PULLER_NAME_KEY = 'puller' PUSHER_NAME_KEY = 'pusher' -API_KEY_KEY = 'api_key' +API_KEY_KEY = 'api-key' HOST_KEY = 'host' LISTENER_ACTOR_KEY = 'listener_actor' diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 9b2ebc82..375ba281 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -466,9 +466,9 @@ def check_k8s_pre_processor_infos(pre_processor: K8sPreProcessorActor, expected_ """ assert isinstance(pre_processor, K8sPreProcessorActor) - assert pre_processor.state.k8s_api_mode == expected_pre_processor_info["k8s_api_mode"] - assert pre_processor.state.time_interval == expected_pre_processor_info["time_interval"] - assert pre_processor.state.timeout_query == expected_pre_processor_info["timeout_query"] + assert pre_processor.state.k8s_api_mode == expected_pre_processor_info["k8s-api-mode"] + assert pre_processor.state.time_interval == expected_pre_processor_info["time-interval"] + assert pre_processor.state.timeout_query == expected_pre_processor_info["timeout-query"] assert len(pre_processor.state.target_actors) == 0 assert len(pre_processor.state.target_actors_names) == 1 assert pre_processor.state.target_actors_names[0] == expected_pre_processor_info["puller"] @@ -519,8 +519,8 @@ def test_generate_k8s_pre_processor_uses_default_values_with_missing_arguments( processors = generator.generate(several_k8s_pre_processors_without_some_arguments_config) - expected_processor_info = {'k8s_api_mode': None, 'time_interval': TIME_INTERVAL_DEFAULT_VALUE, - 'timeout_query': TIMEOUT_QUERY_DEFAULT_VALUE} + expected_processor_info = {'k8s-api-mode': None, 'time-interval': TIME_INTERVAL_DEFAULT_VALUE, + 'timeout-query': TIMEOUT_QUERY_DEFAULT_VALUE} assert len(processors) == len(several_k8s_pre_processors_without_some_arguments_config['pre-processor']) diff --git a/tests/utils/cli/k8s_pre_processor_configuration.json b/tests/utils/cli/k8s_pre_processor_configuration.json index 0f14abf2..3d50a133 100644 --- a/tests/utils/cli/k8s_pre_processor_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_configuration.json @@ -3,9 +3,9 @@ "pre-processor": { "my_processor": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 20, - "timeout_query": 30, + "k8s-api-mode": "Manual", + "time-interval": 20, + "timeout-query": 30, "puller": "my_puller" } } diff --git a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json index 9a266f3d..5f979837 100644 --- a/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_with_non_existing_puller_configuration.json @@ -22,9 +22,9 @@ "pre-processor": { "my_processor": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 50, - "timeout_query": 60, + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, "puller": "non_existent_puller" } } diff --git a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json index ed5918bd..b447bd92 100644 --- a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json @@ -22,16 +22,16 @@ "pre-processor": { "my_processor": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 50, - "timeout_query": 60, + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, "puller": "one_puller" }, "my_processor_2": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 50, - "timeout_query": 60, + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 60, "puller": "one_puller" } } diff --git a/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json index 8751c39d..c3198210 100644 --- a/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json @@ -22,9 +22,9 @@ "pre-processor": { "my_processor": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 70, - "timeout_query": 80, + "k8s-api-mode": "Manual", + "time-interval": 70, + "timeout-query": 80, "puller": "one_pusher" } } diff --git a/tests/utils/cli/several_k8s_pre_processors_configuration.json b/tests/utils/cli/several_k8s_pre_processors_configuration.json index e94418ca..48112c4e 100644 --- a/tests/utils/cli/several_k8s_pre_processors_configuration.json +++ b/tests/utils/cli/several_k8s_pre_processors_configuration.json @@ -3,44 +3,44 @@ "pre-processor": { "my_processor_1": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 20, - "timeout_query": 20, + "k8s-api-mode": "Manual", + "time-interval": 20, + "timeout-query": 20, "puller": "my_puller_1" }, "my_processor_2": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 30, - "timeout_query": 30, + "k8s-api-mode": "Manual", + "time-interval": 30, + "timeout-query": 30, "puller": "my_puller_2" }, "my_processor_3": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 40, - "timeout_query": 40, + "k8s-api-mode": "Manual", + "time-interval": 40, + "timeout-query": 40, "puller": "my_puller_3" }, "my_processor_4": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 50, - "timeout_query": 50, + "k8s-api-mode": "Manual", + "time-interval": 50, + "timeout-query": 50, "puller": "my_puller_4" }, "my_processor_5": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 60, - "timeout_query": 60, + "k8s-api-mode": "Manual", + "time-interval": 60, + "timeout-query": 60, "puller": "my_puller_5" }, "my_processor_6": { "type": "k8s", - "k8s_api_mode": "Manual", - "time_interval": 70, - "timeout_query": 70, + "k8s-api-mode": "Manual", + "time-interval": 70, + "timeout-query": 70, "puller": "my_puller_6" } } From 767790cea33f04cdecb53618a7e4ae43b263c15e Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 2 Nov 2023 16:42:13 +0100 Subject: [PATCH 23/35] refactor: Add support to BackendSupervisor to consider pre-processors when killing actors in postmorten mode --- .../backend_supervisor/backend_supervisor.py | 12 ++++++++++-- powerapi/processor/pre/k8s/k8s_monitor.py | 18 +++++++++--------- .../processor/pre/k8s/test_k8s_processor.py | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/powerapi/backend_supervisor/backend_supervisor.py b/powerapi/backend_supervisor/backend_supervisor.py index aa31b558..50da90f6 100644 --- a/powerapi/backend_supervisor/backend_supervisor.py +++ b/powerapi/backend_supervisor/backend_supervisor.py @@ -32,10 +32,10 @@ from powerapi.actor import Supervisor from powerapi.puller import PullerActor from powerapi.dispatcher import DispatcherActor +from powerapi.pusher import PusherActor class BackendSupervisor(Supervisor): - """ Provide additional functionality to deal with actors: join """ @@ -55,6 +55,9 @@ def __init__(self, stream_mode): #: (list): List of Pusher self.pushers = [] + #: (list): List of pre processors + self.pre_processors = [] + def join(self): """ wait until all actor are terminated @@ -65,8 +68,10 @@ def join(self): self.pullers.append(actor) elif isinstance(actor, DispatcherActor): self.dispatchers.append(actor) - else: + elif isinstance(actor, PusherActor): self.pushers.append(actor) + else: + self.pre_processors.append(actor) if self.stream_mode: self.join_stream_mode_on() @@ -103,6 +108,9 @@ def join_stream_mode_off(self): """ for puller in self.pullers: puller.join() + for pre_processor in self.pre_processors: + pre_processor.soft_kill() + pre_processor.join() for dispatcher in self.dispatchers: dispatcher.soft_kill() dispatcher.join() diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py index 367fb5cb..5666eb8c 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -183,8 +183,8 @@ def query_k8s(self): """ Query k8s for changes and update the metadata cache """ - while not self.stop_monitoring.is_set(): - try: + try: + while not self.stop_monitoring.is_set(): events = self.k8s_streaming_query() for event in events: event_type, namespace, pod_name, container_ids, labels = event @@ -200,13 +200,13 @@ def query_k8s(self): self.stop_monitoring.wait(timeout=self.concerned_actor_state.time_interval) - except BrokenPipeError: - # This error can happen when stopping the monitor process - return - except Exception as ex: - self.logger.warning("Failed streaming query %s", ex) - finally: - self.manager.shutdown() + except BrokenPipeError: + # This error can happen when stopping the monitor process + return + except Exception as ex: + self.logger.warning("Failed streaming query %s", ex) + finally: + self.manager.shutdown() def k8s_streaming_query(self) -> list: """ diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py index b1278ba4..4318d7c9 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -288,7 +288,7 @@ def test_update_metadata_cache_with_modified_event(self, mocked_monitor_modified metadata_for_update = get_metadata_from_event(basic_event=basic_modified_event_k8s) mocked_monitor_modified_event.start() - sleep(2) + sleep(3) result = started_actor.state.metadata_cache_manager From 2c35cb94ce4bbffd6ce5175e94d8356595e2f85c Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 6 Nov 2023 10:31:37 +0100 Subject: [PATCH 24/35] docs: Update BackendSupervisor documentation --- powerapi/backend_supervisor/backend_supervisor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/powerapi/backend_supervisor/backend_supervisor.py b/powerapi/backend_supervisor/backend_supervisor.py index 50da90f6..b761dcc9 100644 --- a/powerapi/backend_supervisor/backend_supervisor.py +++ b/powerapi/backend_supervisor/backend_supervisor.py @@ -102,6 +102,7 @@ def join_stream_mode_off(self): """ Supervisor behaviour when stream mode is off. - Supervisor wait the Puller death + - Supervisor wait the pre-processors death - Supervisor wait for the dispatcher death - Supervisor send a PoisonPill (by_data) to the Pusher - Supervisor wait for the Pusher death From 8866117eec60e424cd0208a28f32753496c148f1 Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 6 Nov 2023 13:43:43 +0100 Subject: [PATCH 25/35] refactor: Remove unnecessary code --- .../processor/pre/k8s/k8s_monitor_handlers.py | 61 ------------------- .../pre/k8s/k8s_pre_processor_handlers.py | 13 ---- 2 files changed, 74 deletions(-) delete mode 100644 powerapi/processor/pre/k8s/k8s_monitor_handlers.py diff --git a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py b/powerapi/processor/pre/k8s/k8s_monitor_handlers.py deleted file mode 100644 index 1079b72e..00000000 --- a/powerapi/processor/pre/k8s/k8s_monitor_handlers.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2023, INRIA -# Copyright (c) 2023, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from threading import Thread - -from powerapi.actor import State -from powerapi.handler import StartHandler, PoisonPillMessageHandler - - -class K8sMonitorAgentStartMessageHandler(StartHandler): - """ - Start the K8sMonitorAgent - """ - - def __init__(self, state: State): - StartHandler.__init__(self, state=state) - - def initialization(self): - self.state.active_monitoring = True - self.state.listener_agent.connect_data() - self.state.monitor_thread = Thread(target=self.state.actor.query_k8s) - self.state.monitor_thread.start() - - -class K8sMonitorAgentPoisonPillMessageHandler(PoisonPillMessageHandler): - """ - Stop the K8sMonitorAgent - """ - - def __init__(self, state: State): - PoisonPillMessageHandler.__init__(self, state=state) - - def teardown(self, soft=False): - self.state.actor.logger.debug('teardown K8sMonitorAgent') - self.state.active_monitoring = False - self.state.monitor_thread.join(10) diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index 39cf2fa4..77ef197e 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -95,19 +95,6 @@ def teardown(self, soft=False): actor.close() -class K8sPreProcessorActorK8sPodUpdateMessageHandler(Handler): - """ - Process the K8sPodUpdateMessage - """ - - def __init__(self, state: State): - Handler.__init__(self, state=state) - - def handle(self, message: Message): - self.state.actor.logger.debug(f"received K8sPodUpdateMessage message {message}") - self.state.metadata_cache_manager.update_cache(message) - - def clean_up_container_id(c_id): """ On some system, we receive a container id that requires some cleanup to match From a9e232ae9928e9f1e0f50c7b0c8ccd26834ad796 Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 6 Nov 2023 16:10:42 +0100 Subject: [PATCH 26/35] refactor: Remove Report Modifiers fonctionality. They are replaced by pre-processors --- powerapi/cli/common_cli_parsing_manager.py | 24 +--- powerapi/cli/generator.py | 29 +---- powerapi/message.py | 1 - .../pre/k8s/k8s_pre_processor_handlers.py | 2 +- powerapi/puller/handlers.py | 12 +- powerapi/puller/puller_actor.py | 9 +- powerapi/report_modifier/__init__.py | 31 ------ powerapi/report_modifier/libvirt_mapper.py | 68 ------------ powerapi/report_modifier/report_modifier.py | 41 ------- ...simple_architecture_with_libvirt_mapper.py | 104 ------------------ ...with_sigterm_signal_must_stop_all_actor.py | 2 +- tests/unit/cli/test_generator.py | 16 +-- .../processor/pre/k8s/test_k8s_processor.py | 2 +- .../report_modifier/test_libvirt_mapper.py | 68 ------------ tests/utils/acceptation.py | 11 +- tests/utils/libvirt.py | 6 +- 16 files changed, 27 insertions(+), 399 deletions(-) delete mode 100644 powerapi/report_modifier/__init__.py delete mode 100644 powerapi/report_modifier/libvirt_mapper.py delete mode 100644 powerapi/report_modifier/report_modifier.py delete mode 100644 tests/acceptation/test_simple_architecture_with_libvirt_mapper.py delete mode 100644 tests/unit/report_modifier/test_libvirt_mapper.py diff --git a/powerapi/cli/common_cli_parsing_manager.py b/powerapi/cli/common_cli_parsing_manager.py index a6a63b3b..8b95eb77 100644 --- a/powerapi/cli/common_cli_parsing_manager.py +++ b/powerapi/cli/common_cli_parsing_manager.py @@ -38,9 +38,7 @@ POWERAPI_OUTPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'OUTPUT_' POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'INPUT_' POWERAPI_PRE_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'PRE_PROCESSOR_' -POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'POST_PROCESSOR_' -POWERAPI_BINDING_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'BINDING_' -POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'REPORT_MODIFIER_' +POWERAPI_POST_PROCESSOR_ENVIRONMENT_VARIABLE_PREFIX = POWERAPI_ENVIRONMENT_VARIABLE_PREFIX + 'POST_PROCESSOR' def extract_file_names(arg, val, args, acc): @@ -63,11 +61,6 @@ def __init__(self): self.add_argument_prefix(argument_prefix=POWERAPI_ENVIRONMENT_VARIABLE_PREFIX) # Subgroups - self.add_subgroup(name='report_modifier', - prefix=POWERAPI_REPORT_MODIFIER_ENVIRONMENT_VARIABLE_PREFIX, - help_text="Specify a report modifier to change input report values : " - "--report_modifier ARG1 ARG2 ...") - self.add_subgroup(name='input', prefix=POWERAPI_INPUT_ENVIRONMENT_VARIABLE_PREFIX, help_text="specify a database input : --input database_name ARG1 ARG2 ... ") @@ -103,21 +96,6 @@ def __init__(self): help_text="enable stream mode", ) - subparser_libvirt_mapper_modifier = SubgroupConfigParsingManager("libvirt_mapper") - subparser_libvirt_mapper_modifier.add_argument( - "u", "uri", help_text="libvirt daemon uri", default_value="" - ) - subparser_libvirt_mapper_modifier.add_argument( - "d", - "domain_regexp", - help_text="regexp used to extract domain from cgroup string", - ) - subparser_libvirt_mapper_modifier.add_argument("n", "name", help_text="") - self.add_subgroup_parser( - subgroup_name="report_modifier", - subgroup_parser=subparser_libvirt_mapper_modifier - ) - subparser_mongo_input = SubgroupConfigParsingManager("mongodb") subparser_mongo_input.add_argument("u", "uri", help_text="specify MongoDB uri") subparser_mongo_input.add_argument( diff --git a/powerapi/cli/generator.py b/powerapi/cli/generator.py index ae50edf0..020f9f6b 100644 --- a/powerapi/cli/generator.py +++ b/powerapi/cli/generator.py @@ -47,7 +47,6 @@ from powerapi.puller import PullerActor from powerapi.pusher import PusherActor -from powerapi.report_modifier.libvirt_mapper import LibvirtMapper from powerapi.puller.simple.simple_puller_actor import SimplePullerActor from powerapi.pusher.simple.simple_pusher_actor import SimplePusherActor @@ -270,18 +269,13 @@ class PullerGenerator(DBActorGenerator): Generate Puller Actor class and Puller start message from config """ - def __init__(self, report_filter: Filter, report_modifier_list=None): + def __init__(self, report_filter: Filter): DBActorGenerator.__init__(self, 'input') self.report_filter = report_filter - if report_modifier_list is None: - report_modifier_list = [] - self.report_modifier_list = report_modifier_list - def _actor_factory(self, actor_name: str, main_config, component_config: dict): return PullerActor(name=actor_name, database=component_config[COMPONENT_DB_MANAGER_KEY], report_filter=self.report_filter, stream_mode=main_config[GENERAL_CONF_STREAM_MODE_KEY], - report_modifier_list=self.report_modifier_list, report_model=component_config[COMPONENT_MODEL_KEY], level_logger=logging.DEBUG if main_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO) @@ -343,27 +337,6 @@ def _actor_factory(self, actor_name: str, main_config: dict, component_config: d number_of_reports_to_store=component_config['number_of_reports_to_store']) -class ReportModifierGenerator: - """ - Generate Report modifier list from config - """ - - def __init__(self): - self.factory = {'libvirt_mapper': lambda config: LibvirtMapper(config['uri'], config['domain_regexp'])} - - def generate(self, config: dict): - """ - Generate Report modifier list from config - """ - report_modifier_list = [] - if 'report_modifier' not in config: - return [] - for report_modifier_name in config['report_modifier'].keys(): - report_modifier = self.factory[report_modifier_name](config['report_modifier'][report_modifier_name]) - report_modifier_list.append(report_modifier) - return report_modifier_list - - class ProcessorGenerator(Generator): """ Generator that initialises the processor from config diff --git a/powerapi/message.py b/powerapi/message.py index 1ec910bf..cd61897f 100644 --- a/powerapi/message.py +++ b/powerapi/message.py @@ -36,7 +36,6 @@ from powerapi.filter import Filter from powerapi.dispatcher import RouteTable from powerapi.formula import FormulaActor, FormulaState - from powerapi.report_modifier import ReportModifier class Message: diff --git a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py index 77ef197e..16aca2a2 100644 --- a/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py +++ b/powerapi/processor/pre/k8s/k8s_pre_processor_handlers.py @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from powerapi.actor import State -from powerapi.handler import Handler, StartHandler, PoisonPillMessageHandler +from powerapi.handler import StartHandler, PoisonPillMessageHandler from powerapi.message import Message from powerapi.processor.handlers import ProcessorReportHandler diff --git a/powerapi/puller/handlers.py b/powerapi/puller/handlers.py index 24817a19..5535c76f 100644 --- a/powerapi/puller/handlers.py +++ b/powerapi/puller/handlers.py @@ -87,17 +87,12 @@ def _pull_database(self): def _get_dispatchers(self, report): return self.state.report_filter.route(report) - def _modify_report(self, report): - for report_modifier in self.state.report_modifier_list: - report = report_modifier.modify_report(report) - return report - def run(self): """ Read data from Database and send it to the dispatchers. If there is no more data, send a kill message to every dispatcher. - If stream mode is disable, kill the actor. + If stream mode is disabled, kill the actor. :param None msg: None. """ @@ -113,11 +108,10 @@ def run(self): while self.state.alive: try: raw_report = self._pull_database() - report = self._modify_report(raw_report) - dispatchers = self._get_dispatchers(report) + dispatchers = self._get_dispatchers(raw_report) for dispatcher in dispatchers: - dispatcher.send_data(report) + dispatcher.send_data(raw_report) except NoReportExtractedException: time.sleep(self.state.timeout_puller / 1000) diff --git a/powerapi/puller/puller_actor.py b/powerapi/puller/puller_actor.py index 0a724319..0c2454df 100644 --- a/powerapi/puller/puller_actor.py +++ b/powerapi/puller/puller_actor.py @@ -51,7 +51,7 @@ class PullerState(State): """ def __init__(self, actor: Actor, database, report_filter, report_model, stream_mode, timeout_puller, - report_modifier_list=[], asynchrone=False): + asynchrone=False): """ :param BaseDB database: Allow to interact with a Database :param Filter report_filter: Filter of the Puller @@ -84,8 +84,6 @@ def __init__(self, actor: Actor, database, report_filter, report_model, stream_m self.loop = None - self.report_modifier_list = report_modifier_list - class PullerActor(Actor): """ @@ -95,7 +93,7 @@ class PullerActor(Actor): to many Dispatcher depending on some rules. """ - def __init__(self, name, database, report_filter, report_model, stream_mode=False, report_modifier_list=[], + def __init__(self, name, database, report_filter, report_model, stream_mode=False, level_logger=logging.WARNING, timeout=5000, timeout_puller=100): """ @@ -110,8 +108,7 @@ def __init__(self, name, database, report_filter, report_model, stream_mode=Fals Actor.__init__(self, name, level_logger, timeout) #: (State): Actor State. self.state = PullerState(self, database=database, report_filter=report_filter, report_model=report_model, - stream_mode=stream_mode, timeout_puller=timeout_puller, - report_modifier_list=report_modifier_list, asynchrone=database.asynchrone) + stream_mode=stream_mode, timeout_puller=timeout_puller, asynchrone=database.asynchrone) self.low_exception += database.exceptions diff --git a/powerapi/report_modifier/__init__.py b/powerapi/report_modifier/__init__.py deleted file mode 100644 index e9faee94..00000000 --- a/powerapi/report_modifier/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.report_modifier.report_modifier import ReportModifier -from powerapi.report_modifier.libvirt_mapper import LibvirtMapper diff --git a/powerapi/report_modifier/libvirt_mapper.py b/powerapi/report_modifier/libvirt_mapper.py deleted file mode 100644 index ff30112e..00000000 --- a/powerapi/report_modifier/libvirt_mapper.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import re -import logging - -try: - from libvirt import openReadOnly, libvirtError -except ImportError: - logging.getLogger().info("libvirt-python is not installed.") - - class LibvirtException(Exception): - """""" - def __init__(self, _): - Exception.__init__(self) - - libvirtError = LibvirtException - openReadOnly = None - -from powerapi.report_modifier.report_modifier import ReportModifier - - -class LibvirtMapper(ReportModifier): - """ - Report modifier which modifi target with libvirt id by open stak uuid - """ - - def __init__(self, uri: str, regexp: str): - self.regexp = re.compile(regexp) - daemon_uri = None if uri == '' else uri - self.libvirt = openReadOnly(daemon_uri) - - def modify_report(self, report): - result = re.match(self.regexp, report.target) - if result is not None: - domain_name = result.groups(0)[0] - try: - domain = self.libvirt.lookupByName(domain_name) - report.metadata["domain_id"] = domain.UUIDString() - except libvirtError: - pass - return report diff --git a/powerapi/report_modifier/report_modifier.py b/powerapi/report_modifier/report_modifier.py deleted file mode 100644 index 9a441327..00000000 --- a/powerapi/report_modifier/report_modifier.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from powerapi.report import Report - - -class ReportModifier: - """ - Abstract class for object used by puller to modify reports before sending them to dispatcher - """ - def modify_report(self, report: Report) -> Report: - """ - modify the given report - """ - raise NotImplementedError() diff --git a/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py b/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py deleted file mode 100644 index 39922ea7..00000000 --- a/tests/acceptation/test_simple_architecture_with_libvirt_mapper.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -""" -Test the behaviour of the most simple architecture with a libvirt mapper - -Architecture : - - 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off, with a report_modifier (LibvirtMapper)) - - 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET) - - 1 Dummy Formula - - 1 pusher (connected to MongoDB1 [collection test_result] - -MongoDB1 content: -- 10 HWPCReports with 2 socket for target LIBVIRT_TARGET_NAME1 -- 10 HWPCReports with 2 socket for target LIBVIRT_TARGET_NAME2 - -Scenario: - - Launch the full architecture with a libvirt mapper connected to a fake libvirt daemon which only know LIBVIRT_INSTANCE_NAME1 -Test if: - - each HWPCReport in the intput database was converted in one PowerReport per - socket in the output database - - only target name LIBVIRT_TARGET_NAME1 was converted into UUID_1 -""" -# pylint: disable=redefined-outer-name,disable=unused-argument,disable=unused-import -import time -from datetime import datetime -from mock import patch - -import pytest -import pymongo - -from powerapi.actor import Supervisor -from tests.utils.formula.dummy import DummyFormulaActor -from tests.utils.acceptation import launch_simple_architecture, SOCKET_DEPTH_LEVEL, LIBVIRT_CONFIG -# noinspection PyUnresolvedReferences -from tests.utils.db.mongo import MONGO_URI, MONGO_INPUT_COLLECTION_NAME, MONGO_OUTPUT_COLLECTION_NAME, \ - MONGO_DATABASE_NAME, mongo_database -from tests.utils.report.hwpc import extract_all_events_reports_with_vm_name -from tests.utils.libvirt import MockedLibvirt, LIBVIRT_TARGET_NAME1, UUID_1 - - -@pytest.fixture -def mongodb_content(): - """ - Get reports from a file for testing purposes - """ - return extract_all_events_reports_with_vm_name(20) - - -def check_db(): - """ - Verify that output DB has correct information - """ - mongo = pymongo.MongoClient(MONGO_URI) - c_input = mongo[MONGO_DATABASE_NAME][MONGO_INPUT_COLLECTION_NAME] - c_output = mongo[MONGO_DATABASE_NAME][MONGO_OUTPUT_COLLECTION_NAME] - - assert c_output.count_documents({}) == c_input.count_documents({}) * 2 - for report in c_input.find({"target": LIBVIRT_TARGET_NAME1}): - ts = datetime.strptime(report['timestamp'], "%Y-%m-%dT%H:%M:%S.%f") - for socket_report in c_output.find({'timestamp': ts, 'sensor': report['sensor']}): - assert socket_report["metadata"]["domain_id"] == UUID_1 - - -@patch('powerapi.report_modifier.libvirt_mapper.openReadOnly', return_value=MockedLibvirt()) -def test_run(mocked_libvirt, mongo_database, shutdown_system): - """ - Check that the actor system behave correctly with libvirt mapper - """ - supervisor = Supervisor() - launch_simple_architecture(config=LIBVIRT_CONFIG, supervisor=supervisor, hwpc_depth_level=SOCKET_DEPTH_LEVEL, - formula_class=DummyFormulaActor, generate_report_modifier_list=True) - time.sleep(2) - - check_db() - - supervisor.kill_actors() diff --git a/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py b/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py index 623343e0..255a6d87 100644 --- a/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py +++ b/tests/acceptation/test_stop_architecture_with_sigterm_signal_must_stop_all_actor.py @@ -90,7 +90,7 @@ def term_handler(_, __): launch_simple_architecture(config=get_basic_config_with_stream(), supervisor=self.supervisor, hwpc_depth_level=SOCKET_DEPTH_LEVEL, - formula_class=DummyFormulaActor, generate_report_modifier_list=True) + formula_class=DummyFormulaActor) @pytest.fixture diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 375ba281..9c0728ab 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -54,7 +54,7 @@ def test_generate_puller_from_empty_config_dict_raise_an_exception(): Test that PullerGenerator raises a PowerAPIException when there is no input argument """ conf = {} - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(conf) @@ -64,7 +64,7 @@ def test_generate_puller_from_mongo_basic_config(mongodb_input_output_stream_con """ Test that generation for mongodb puller from a config with a mongodb input works correctly """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(mongodb_input_output_stream_config) @@ -89,7 +89,7 @@ def test_generate_several_pullers_from_config(several_inputs_outputs_stream_conf for _, current_input in several_inputs_outputs_stream_config['input'].items(): if current_input['type'] == 'csv': current_input['files'] = current_input['files'].split(',') - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(several_inputs_outputs_stream_config) assert len(pullers) == len(several_inputs_outputs_stream_config['input']) @@ -123,7 +123,7 @@ def test_generate_puller_raise_exception_when_missing_arguments_in_mongo_input( """ Test that PullerGenerator raise a PowerAPIException when some arguments are missing for mongo input """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(several_inputs_outputs_stream_mongo_without_some_arguments_config) @@ -134,7 +134,7 @@ def test_generate_puller_when_missing_arguments_in_csv_input_generate_related_ac """ Test that PullerGenerator generates the csv related actors even if there are some missing arguments """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) pullers = generator.generate(several_inputs_outputs_stream_csv_without_some_arguments_config) @@ -159,7 +159,7 @@ def test_generate_puller_raise_exception_when_missing_arguments_in_socket_input( """ Test that PullerGenerator raise a PowerAPIException when some arguments are missing for socket input """ - generator = PullerGenerator(report_filter=None, report_modifier_list=[]) + generator = PullerGenerator(report_filter=None) with pytest.raises(PowerAPIException): generator.generate(several_inputs_outputs_stream_socket_without_some_arguments_config) @@ -187,7 +187,7 @@ def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_rep """ Test that PullGenerator raises PowerAPIException when the model class is not defined """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) generator.remove_report_class('HWPCReport') with pytest.raises(PowerAPIException): _ = generator.generate(mongodb_input_output_stream_config) @@ -198,7 +198,7 @@ def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_i """ Test that PullGenerator raises a PowerAPIException when an input type is not defined """ - generator = PullerGenerator(None, []) + generator = PullerGenerator(report_filter=None) generator.remove_db_factory('mongodb') with pytest.raises(PowerAPIException): _ = generator.generate(mongodb_input_output_stream_config) diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py index 4318d7c9..db784de9 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -288,7 +288,7 @@ def test_update_metadata_cache_with_modified_event(self, mocked_monitor_modified metadata_for_update = get_metadata_from_event(basic_event=basic_modified_event_k8s) mocked_monitor_modified_event.start() - sleep(3) + sleep(5) result = started_actor.state.metadata_cache_manager diff --git a/tests/unit/report_modifier/test_libvirt_mapper.py b/tests/unit/report_modifier/test_libvirt_mapper.py deleted file mode 100644 index 9d02aac1..00000000 --- a/tests/unit/report_modifier/test_libvirt_mapper.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2021, INRIA -# Copyright (c) 2021, University of Lille -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from mock import patch - -try: - from libvirt import libvirtError -except ImportError: - libvirtError = Exception - -from powerapi.report_modifier import LibvirtMapper -from powerapi.report import Report - -from tests.utils.libvirt import MockedLibvirt, LIBVIRT_TARGET_NAME1, LIBVIRT_TARGET_NAME2, UUID_1, REGEXP - -BAD_TARGET = 'lkjqlskjdlqksjdlkj' - - -@pytest.fixture -def libvirt_mapper(): - with patch('powerapi.report_modifier.libvirt_mapper.openReadOnly', return_value=MockedLibvirt()): - return LibvirtMapper('', REGEXP) - - -def test_modify_report_that_not_match_regexp_musnt_modify_report(libvirt_mapper): - report = Report(0, 'sensor', BAD_TARGET) - new_report = libvirt_mapper.modify_report(report) - assert new_report.target == BAD_TARGET - assert new_report.metadata == {} - - -def test_modify_report_that_match_regexp_must_modify_report(libvirt_mapper): - report = Report(0, 'sensor', LIBVIRT_TARGET_NAME1) - new_report = libvirt_mapper.modify_report(report) - assert new_report.metadata["domain_id"] == UUID_1 - - -def test_modify_report_that_match_regexp_but_with_wrong_domain_name_musnt_modify_report(libvirt_mapper): - report = Report(0, 'sensor', LIBVIRT_TARGET_NAME2) - new_report = libvirt_mapper.modify_report(report) - assert new_report.metadata == {} diff --git a/tests/utils/acceptation.py b/tests/utils/acceptation.py index aa35052b..5155fef6 100644 --- a/tests/utils/acceptation.py +++ b/tests/utils/acceptation.py @@ -36,7 +36,7 @@ import pytest from powerapi.actor import Supervisor -from powerapi.cli.generator import PusherGenerator, PullerGenerator, ReportModifierGenerator +from powerapi.cli.generator import PusherGenerator, PullerGenerator from powerapi.dispatch_rule import HWPCDispatchRule, HWPCDepthLevel from powerapi.dispatcher import RouteTable, DispatcherActor from powerapi.filter import Filter @@ -134,7 +134,7 @@ def get_basic_config_with_stream(): def launch_simple_architecture(config: Dict, supervisor: Supervisor, hwpc_depth_level: str, - formula_class: Callable, generate_report_modifier_list=False): + formula_class: Callable): """ Launch a simple architecture with a pusher, a dispatcher et a puller. :param config: Architecture configuration @@ -169,12 +169,7 @@ def launch_simple_architecture(config: Dict, supervisor: Supervisor, hwpc_depth_ report_filter = Filter() report_filter.filter(filter_rule, dispatcher) - report_modifier_list = [] - if generate_report_modifier_list: - report_modifier_generator = ReportModifierGenerator() - report_modifier_list = report_modifier_generator.generate(config) - - puller_generator = PullerGenerator(report_filter, report_modifier_list) + puller_generator = PullerGenerator(report_filter) puller_info = puller_generator.generate(config) puller = puller_info['test_puller'] supervisor.launch_actor(actor=puller, start_message=True) diff --git a/tests/utils/libvirt.py b/tests/utils/libvirt.py index 8e0bddf3..4963c6d6 100644 --- a/tests/utils/libvirt.py +++ b/tests/utils/libvirt.py @@ -32,7 +32,11 @@ try: from libvirt import libvirtError except ImportError: - from powerapi.report_modifier.libvirt_mapper import LibvirtException + class LibvirtException(Exception): + """""" + + def __init__(self, _): + Exception.__init__(self) libvirtError = LibvirtException DOMAIN_NAME_1 = 'instance-00000001' From 37cb3b855027ddf0503897d1d57465a66633c1f8 Mon Sep 17 00:00:00 2001 From: daniel Date: Wed, 8 Nov 2023 15:40:31 +0100 Subject: [PATCH 27/35] fix: Add kubernetes dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 02f5d8a0..7d71ef01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ authors = [ dependencies = [ "pyzmq >= 18.1.0", "setproctitle >= 1.1.8", + "kubernetes >= 27.0.2" ] [project.optional-dependencies] From 11c02bd67f30e784061b1efff5f236c948c5d1e0 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 09:44:51 +0100 Subject: [PATCH 28/35] build: Define kuberntes as a group and Add it to all-platforms group --- powerapi/cli/binding_manager.py | 1 + pyproject.toml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/powerapi/cli/binding_manager.py b/powerapi/cli/binding_manager.py index 4e00fe80..aac9a4aa 100644 --- a/powerapi/cli/binding_manager.py +++ b/powerapi/cli/binding_manager.py @@ -26,6 +26,7 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# pylint: disable=R1702 from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor diff --git a/pyproject.toml b/pyproject.toml index 7d71ef01..9f0001ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,7 @@ authors = [ dependencies = [ "pyzmq >= 18.1.0", - "setproctitle >= 1.1.8", - "kubernetes >= 27.0.2" + "setproctitle >= 1.1.8" ] [project.optional-dependencies] @@ -60,10 +59,11 @@ prometheus = ["prometheus-client >= 0.9.0"] # Plaforms: libvirt = ["libvirt-python >= 6.1.0"] # requires libvirt lib/headers, do not include by default. +kubernetes = ["kubernetes >= 27.0.2"] # Aliases: all-databases = ["powerapi[mongodb, influxdb, opentsdb, prometheus]"] -all-platforms = [] +all-platforms = ["kubernetes"] everything = ["powerapi[all-databases, all-platforms]"] devel = ["powerapi[everything, test, docs, lint]"] From 707012375496814051d2712259a5ac413b108c2c Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 12:15:47 +0100 Subject: [PATCH 29/35] refactor: Skip some libvirt tests and remove some of them --- tests/unit/cli/conftest.py | 9 +++------ tests/unit/cli/test_generator.py | 3 +++ tests/unit/processor/pre/k8s/test_k8s_processor.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 1db8670e..1fa5fdfb 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -640,8 +640,7 @@ def empty_pre_processor_config(pre_processor_complete_configuration): return pre_processor_complete_configuration -@pytest.fixture(params=['libvirt_pre_processor_wrong_binding_configuration.json', - 'k8s_pre_processor_wrong_binding_configuration.json']) +@pytest.fixture(params=['k8s_pre_processor_wrong_binding_configuration.json']) def pre_processor_wrong_binding_configuration(request): """ Return a dictionary containing wrong bindings with a pre-processor @@ -649,8 +648,7 @@ def pre_processor_wrong_binding_configuration(request): return load_configuration_from_json_file(file_name=request.param) -@pytest.fixture(params=['libvirt_pre_processor_with_non_existing_puller_configuration.json', - 'k8s_pre_processor_with_non_existing_puller_configuration.json']) +@pytest.fixture(params=['k8s_pre_processor_with_non_existing_puller_configuration.json']) def pre_processor_with_unexisting_puller_configuration(request): """ Return a dictionary containing a pre-processor with a puller that doesn't exist @@ -659,8 +657,7 @@ def pre_processor_with_unexisting_puller_configuration(request): file_name=request.param) -@pytest.fixture(params=['libvirt_pre_processor_with_reused_puller_in_bindings_configuration.json', - 'k8s_pre_processor_with_reused_puller_in_bindings_configuration.json']) +@pytest.fixture(params=['k8s_pre_processor_with_reused_puller_in_bindings_configuration.json']) def pre_processor_with_reused_puller_in_bindings_configuration(request): """ Return a dictionary containing a pre-processor with a puller that doesn't exist diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 9c0728ab..25a41a96 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -413,6 +413,7 @@ def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): generator.generate(conf) +@pytest.mark.skip def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config): """ Test that generation for libvirt pre-processor from a config works correctly @@ -431,6 +432,7 @@ def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config assert isinstance(processor.state.regexp, Pattern) +@pytest.mark.skip def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre_processors_config): """ Test that several libvirt pre-processors are correctly generated @@ -449,6 +451,7 @@ def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre assert isinstance(processors[processor_name].state.regexp, Pattern) +@pytest.mark.skip def test_generate_libvirt_pre_processor_raise_exception_when_missing_arguments( several_libvirt_processors_without_some_arguments_config): """ diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py index db784de9..e1080a9c 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -288,7 +288,7 @@ def test_update_metadata_cache_with_modified_event(self, mocked_monitor_modified metadata_for_update = get_metadata_from_event(basic_event=basic_modified_event_k8s) mocked_monitor_modified_event.start() - sleep(5) + sleep(10) result = started_actor.state.metadata_cache_manager From 4edebcd97e20040dbe2e47a9ccfc6ca2e40f3808 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 12:35:20 +0100 Subject: [PATCH 30/35] refactor: Skip some libvirt tests --- tests/unit/processor/pre/libvirt/test_libvirt_processor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/processor/pre/libvirt/test_libvirt_processor.py b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py index 2a731807..ed309d75 100644 --- a/tests/unit/processor/pre/libvirt/test_libvirt_processor.py +++ b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py @@ -68,6 +68,7 @@ def actor(self, started_fake_dispatcher): return LibvirtPreProcessorActor(name='processor_actor', uri='', regexp=REGEXP, target_actors=[started_fake_dispatcher]) + @pytest.mark.skip def test_modify_report_that_not_match_regexp_must_not_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): @@ -79,6 +80,7 @@ def test_modify_report_that_not_match_regexp_must_not_modify_report(self, starte sleep(1) assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) + @pytest.mark.skip def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): """ Test that a report matching the regexp of the processor is actually modified @@ -88,6 +90,7 @@ def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, new_report = recv_from_pipe(dummy_pipe_out, 2)[1] assert new_report.metadata["domain_id"] == UUID_1 + @pytest.mark.skip def test_modify_report_that_match_regexp_but_with_wrong_domain_name_must_not_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): From 9b428aaa62326628cbab0da6799255cf8d08bc0d Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 14:10:52 +0100 Subject: [PATCH 31/35] refactor: Remove some libvirt tests --- tests/unit/cli/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 1fa5fdfb..fa2657ad 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -609,8 +609,7 @@ def output_input_configuration(): return load_configuration_from_json_file(file_name='output_input_configuration.json') -@pytest.fixture(params=['libvirt_pre_processor_complete_configuration.json', - 'k8s_pre_processor_complete_configuration.json']) +@pytest.fixture(params=['k8s_pre_processor_complete_configuration.json']) def pre_processor_complete_configuration(request): """ Return a dictionary containing a configuration with pre-processor From 3b26eab18c0322700e1d9310c786472f057a76d4 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 14:34:53 +0100 Subject: [PATCH 32/35] refactor: Remove unnecessary code --- tests/unit/cli/conftest.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index fa2657ad..959b184c 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -221,23 +221,6 @@ def several_k8s_pre_processors_without_some_arguments_config(): file_name='several_k8s_pre_processors_without_some_arguments_configuration.json') -def generate_k8s_monitors_config(processors_config): - """ - Generate the configuration related to the given k8s processors configuration - """ - generator = ProcessorGenerator() - - processors = generator.generate(processors_config) - - monitors = {'monitor': {}} - - for processor_name, processor in processors.items(): - monitors['monitor'][processor_name + MONITOR_NAME_SUFFIX] = {COMPONENT_TYPE_KEY: 'k8s', - LISTENER_ACTOR_KEY: processor} - - return monitors - - @pytest.fixture def several_k8s_pre_processors(several_k8s_pre_processors_config): """ From 0225dd5ee19b1cad4b9996c4845e6daabd2c70df Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 15:08:45 +0100 Subject: [PATCH 33/35] refactor: Improve code quality --- powerapi/processor/pre/k8s/k8s_monitor.py | 2 +- tests/unit/cli/conftest.py | 4 +-- tests/unit/cli/test_binding_manager.py | 3 -- tests/unit/cli/test_config_validator.py | 4 --- tests/unit/cli/test_generator.py | 8 ++--- .../processor/pre/k8s/test_k8s_processor.py | 30 +++++++++++-------- .../pre/libvirt/test_libvirt_processor.py | 6 ++-- 7 files changed, 28 insertions(+), 29 deletions(-) diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py index 5666eb8c..4d6b001c 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -227,7 +227,7 @@ def k8s_streaming_query(self) -> list: if event: - if not event["type"] in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: + if event["type"] not in {DELETED_EVENT, ADDED_EVENT, MODIFIED_EVENT}: self.logger.warning( "UNKNOWN EVENT TYPE : %s : %s %s", event['type'], event['object'].metadata.name, event diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 959b184c..6ca32ec7 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -712,7 +712,7 @@ def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_b """ Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + _, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( configuration=pre_processor_wrong_binding_configuration) return PreProcessorBindingManager(pullers=pushers, processors=processors) @@ -735,7 +735,7 @@ def pre_processor_binding_manager_with_reused_puller_in_bindings( """ Return a PreProcessorBindingManager with a puller used by two different pre-processors """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( configuration=pre_processor_with_reused_puller_in_bindings_configuration) return PreProcessorBindingManager(pullers=pullers, processors=processors) diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py index 02bc59e9..e1bf1b24 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/cli/test_binding_manager.py @@ -130,7 +130,6 @@ def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindi """ try: pre_processor_binding_manager.check_processors_targets_are_unique() - assert True except TargetActorAlreadyUsed: assert False @@ -164,8 +163,6 @@ def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding try: for _, processor in pre_processor_binding_manager.processors.items(): pre_processor_binding_manager.check_processor_targets(processor=processor) - - assert True except UnsupportedActorTypeException: assert False except UnexistingActorException: diff --git a/tests/unit/cli/test_config_validator.py b/tests/unit/cli/test_config_validator.py index 4768664d..2d180312 100644 --- a/tests/unit/cli/test_config_validator.py +++ b/tests/unit/cli/test_config_validator.py @@ -161,8 +161,6 @@ def test_validation_of_correct_configuration_with_pre_processors(pre_processor_c except Exception: assert False - assert True - def test_validation_of_correct_configuration_without_pre_processors_and_bindings(output_input_configuration): """ @@ -172,5 +170,3 @@ def test_validation_of_correct_configuration_without_pre_processors_and_bindings ConfigValidator.validate(output_input_configuration) except Exception: assert False - - assert True diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 25a41a96..4e640e0f 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -182,7 +182,7 @@ def test_remove_model_factory_that_does_not_exist_on_a_DBActorGenerator_must_rai assert len(generator.report_classes) == 5 -def test_remove_HWPCReport_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception( +def test_remove_hwpc_report_model_and_generate_puller_from_a_config_with_hwpc_report_model_raise_an_exception( mongodb_input_output_stream_config): """ Test that PullGenerator raises PowerAPIException when the model class is not defined @@ -413,7 +413,7 @@ def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): generator.generate(conf) -@pytest.mark.skip +@pytest.mark.skip(reason='libvirt is disable by default') def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config): """ Test that generation for libvirt pre-processor from a config works correctly @@ -432,7 +432,7 @@ def test_generate_pre_processor_from_libvirt_config(libvirt_pre_processor_config assert isinstance(processor.state.regexp, Pattern) -@pytest.mark.skip +@pytest.mark.skip(reason='libvirt is disable by default') def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre_processors_config): """ Test that several libvirt pre-processors are correctly generated @@ -451,7 +451,7 @@ def test_generate_several_libvirt_pre_processors_from_config(several_libvirt_pre assert isinstance(processors[processor_name].state.regexp, Pattern) -@pytest.mark.skip +@pytest.mark.skip(reason='libvirt is disable by default') def test_generate_libvirt_pre_processor_raise_exception_when_missing_arguments( several_libvirt_processors_without_some_arguments_config): """ diff --git a/tests/unit/processor/pre/k8s/test_k8s_processor.py b/tests/unit/processor/pre/k8s/test_k8s_processor.py index e1080a9c..e7be41a4 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_processor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_processor.py @@ -48,6 +48,12 @@ DISPATCHER_NAME = 'test_k8s_processor_dispatcher' +KUBERNETES_CLIENT_API_REFERENCE = 'kubernetes.client.CoreV1Api' + +KUBERNETES_LOAD_CONFIG_REFERENCE = 'kubernetes.config.load_kube_config' + +KUBERNETES_WATCH_REFERENCE = 'kubernetes.watch.Watch' + def get_metadata_from_event(basic_event: dict): """ @@ -213,10 +219,10 @@ def mocked_monitor_added_event(self, actor, basic_added_event_k8s, pods_list): """ Return a mocked monitor that produces an added event """ - with patch('kubernetes.client.CoreV1Api', + with patch(KUBERNETES_CLIENT_API_REFERENCE, return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): - with patch('kubernetes.config.load_kube_config', return_value=Mock()): - with patch('kubernetes.watch.Watch', + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, return_value=MockedWatch(events=[basic_added_event_k8s])): yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', concerned_actor_state=actor.state) @@ -226,10 +232,10 @@ def mocked_monitor_modified_event(self, actor, basic_modified_event_k8s, pods_li """ Return a mocked monitor that produces a modified event """ - with patch('kubernetes.client.CoreV1Api', + with patch(KUBERNETES_CLIENT_API_REFERENCE, return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): - with patch('kubernetes.config.load_kube_config', return_value=Mock()): - with patch('kubernetes.watch.Watch', + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, return_value=MockedWatch(events=[basic_modified_event_k8s])): yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', concerned_actor_state=actor.state) @@ -239,10 +245,10 @@ def mocked_monitor_deleted_event(self, actor, basic_deleted_event_k8s, pods_list """ Return a mocked monitor that produces a deleted event """ - with patch('kubernetes.client.CoreV1Api', + with patch(KUBERNETES_CLIENT_API_REFERENCE, return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): - with patch('kubernetes.config.load_kube_config', return_value=Mock()): - with patch('kubernetes.watch.Watch', + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, return_value=MockedWatch(events=[basic_deleted_event_k8s])): yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', concerned_actor_state=actor.state) @@ -252,10 +258,10 @@ def mocked_monitor_unknown_event(self, actor, basic_added_event_k8s, basic_unkno """ Return a mocked monitor that produces an unknown event """ - with patch('kubernetes.client.CoreV1Api', + with patch(KUBERNETES_CLIENT_API_REFERENCE, return_value=Mock(list_pod_for_all_namespaces=Mock(return_value=pods_list))): - with patch('kubernetes.config.load_kube_config', return_value=Mock()): - with patch('kubernetes.watch.Watch', + with patch(KUBERNETES_LOAD_CONFIG_REFERENCE, return_value=Mock()): + with patch(KUBERNETES_WATCH_REFERENCE, return_value=MockedWatch(events=[basic_unknown_event_k8s, basic_added_event_k8s])): yield K8sMonitorAgent(name='test_update_metadata_cache_with_added_event_monitor_agent', concerned_actor_state=actor.state) diff --git a/tests/unit/processor/pre/libvirt/test_libvirt_processor.py b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py index ed309d75..2fa21801 100644 --- a/tests/unit/processor/pre/libvirt/test_libvirt_processor.py +++ b/tests/unit/processor/pre/libvirt/test_libvirt_processor.py @@ -68,7 +68,7 @@ def actor(self, started_fake_dispatcher): return LibvirtPreProcessorActor(name='processor_actor', uri='', regexp=REGEXP, target_actors=[started_fake_dispatcher]) - @pytest.mark.skip + @pytest.mark.skip(reason='libvirt is disable by default') def test_modify_report_that_not_match_regexp_must_not_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): @@ -80,7 +80,7 @@ def test_modify_report_that_not_match_regexp_must_not_modify_report(self, starte sleep(1) assert recv_from_pipe(dummy_pipe_out, 2) == (DISPATCHER_NAME, report) - @pytest.mark.skip + @pytest.mark.skip(reason='libvirt is disable by default') def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): """ Test that a report matching the regexp of the processor is actually modified @@ -90,7 +90,7 @@ def test_modify_report_that_match_regexp_must_modify_report(self, started_actor, new_report = recv_from_pipe(dummy_pipe_out, 2)[1] assert new_report.metadata["domain_id"] == UUID_1 - @pytest.mark.skip + @pytest.mark.skip(reason='libvirt is disable by default') def test_modify_report_that_match_regexp_but_with_wrong_domain_name_must_not_modify_report(self, started_actor, dummy_pipe_out, shutdown_system): From 9d672c97920342c6c95a0830345ae3888720c876 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 15:11:10 +0100 Subject: [PATCH 34/35] refactor: Improve code quality --- tests/unit/cli/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 6ca32ec7..2ac51a52 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -723,7 +723,7 @@ def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unex """ Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) """ - pullers, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( + pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( configuration=pre_processor_with_unexisting_puller_configuration) return PreProcessorBindingManager(pullers=pullers, processors=processors) From f4a7765884d91774963b2f6e4601eeaeca2d22b7 Mon Sep 17 00:00:00 2001 From: daniel Date: Thu, 9 Nov 2023 15:37:59 +0100 Subject: [PATCH 35/35] refactor: Improve code quality --- powerapi/processor/pre/k8s/k8s_monitor.py | 16 +++++++--------- tests/unit/cli/test_generator.py | 7 +++---- tests/unit/processor/pre/k8s/test_k8s_monitor.py | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/powerapi/processor/pre/k8s/k8s_monitor.py b/powerapi/processor/pre/k8s/k8s_monitor.py index 4d6b001c..affc1929 100644 --- a/powerapi/processor/pre/k8s/k8s_monitor.py +++ b/powerapi/processor/pre/k8s/k8s_monitor.py @@ -29,10 +29,8 @@ # pylint: disable=W0603,W0718 -import logging -import multiprocessing -from logging import Logger -from multiprocessing import Process +from logging import Formatter, getLogger, Logger, StreamHandler, WARNING +from multiprocessing import Manager, Process from kubernetes import client, config, watch from kubernetes.client.configuration import Configuration @@ -144,7 +142,7 @@ class K8sMonitorAgent(Process): when pod are created, removed or modified. """ - def __init__(self, name: str, concerned_actor_state: K8sPreProcessorState, level_logger: int = logging.WARNING): + def __init__(self, name: str, concerned_actor_state: K8sPreProcessorState, level_logger: int = WARNING): """ :param str name: The actor name :param K8sPreProcessorState concerned_actor_state: state of the actor that will use the monitored information @@ -153,17 +151,17 @@ def __init__(self, name: str, concerned_actor_state: K8sPreProcessorState, level Process.__init__(self, name=name) #: (logging.Logger): Logger - self.logger = logging.getLogger(name) + self.logger = getLogger(name) self.logger.setLevel(level_logger) - formatter = logging.Formatter('%(asctime)s || %(levelname)s || ' + '%(process)d %(processName)s || %(message)s') - handler = logging.StreamHandler() + formatter = Formatter('%(asctime)s || %(levelname)s || ' + '%(process)d %(processName)s || %(message)s') + handler = StreamHandler() handler.setFormatter(formatter) # Concerned Actor state self.concerned_actor_state = concerned_actor_state # Multiprocessing Manager - self.manager = multiprocessing.Manager() + self.manager = Manager() # Shared cache self.concerned_actor_state.metadata_cache_manager = K8sMetadataCacheManager(process_manager=self.manager, diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 4e640e0f..9295e7ae 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -28,12 +28,11 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import re -from re import Pattern +from re import compile, Pattern import pytest -from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, ProcessorGenerator, \ +from powerapi.cli.generator import PullerGenerator, DBActorGenerator, PusherGenerator, \ MonitorGenerator, MONITOR_NAME_SUFFIX, LISTENER_ACTOR_KEY, PreProcessorGenerator from powerapi.cli.generator import ModelNameDoesNotExist from powerapi.processor.pre.k8s.k8s_monitor import K8sMonitorAgent @@ -292,7 +291,7 @@ def test_generate_several_pushers_from_config(several_inputs_outputs_stream_conf assert db.metric_name == current_pusher_infos['metric_name'] elif pusher_type == 'virtiofs': - assert db.vm_name_regexp == re.compile(current_pusher_infos['vm_name_regexp']) + assert db.vm_name_regexp == compile(current_pusher_infos['vm_name_regexp']) assert db.root_directory_name == current_pusher_infos['root_directory_name'] assert db.vm_directory_name_prefix == current_pusher_infos['vm_directory_name_prefix'] assert db.vm_directory_name_suffix == current_pusher_infos['vm_directory_name_suffix'] diff --git a/tests/unit/processor/pre/k8s/test_k8s_monitor.py b/tests/unit/processor/pre/k8s/test_k8s_monitor.py index 65413756..1a251635 100644 --- a/tests/unit/processor/pre/k8s/test_k8s_monitor.py +++ b/tests/unit/processor/pre/k8s/test_k8s_monitor.py @@ -1,7 +1,6 @@ # Copyright (c) 2023, INRIA # Copyright (c) 2023, University of Lille # All rights reserved. -from time import sleep # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -29,6 +28,7 @@ # pylint: disable=R6301,W0613,W0221 +from time import sleep from unittest.mock import patch, Mock import pytest