From 1765970c8c779291cb8c41562ca3e196812c3f1f Mon Sep 17 00:00:00 2001 From: anmolbabu Date: Sun, 11 Dec 2016 12:40:44 +0530 Subject: [PATCH 1/4] Add alert socket and alert handlers tendrl-bug-id: Tendrl/node_agent#70 tendrl-spec: Tendrl/specifications#18 Signed-off-by: anmolbabu --- tendrl/node_agent/alerts/__init__.py | 0 tendrl/node_agent/alerts/alert_socket.py | 70 +++++++++++ .../node_agent/alerts/base_alert_handler.py | 118 ++++++++++++++++++ tendrl/node_agent/alerts/handlers/__init__.py | 0 .../node_agent/alerts/handlers/cpu_handler.py | 7 ++ .../alerts/handlers/memory_handler.py | 7 ++ .../alerts/handlers/mount_point_handler.py | 7 ++ .../alerts/handlers/swap_handler.py | 7 ++ tendrl/node_agent/manager/manager.py | 20 ++- .../manager/tendrl_definitions_node_agent.py | 10 ++ tendrl/node_agent/objects/node/atoms/cmd.py | 3 +- tendrl/node_agent/objects/service/__init__.py | 0 .../objects/service/atoms/__init__.py | 0 .../service/atoms/check_service_status.py | 12 ++ tendrl/node_agent/tests/test_alert_manager.py | 60 +++++++++ 15 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 tendrl/node_agent/alerts/__init__.py create mode 100644 tendrl/node_agent/alerts/alert_socket.py create mode 100644 tendrl/node_agent/alerts/base_alert_handler.py create mode 100644 tendrl/node_agent/alerts/handlers/__init__.py create mode 100644 tendrl/node_agent/alerts/handlers/cpu_handler.py create mode 100644 tendrl/node_agent/alerts/handlers/memory_handler.py create mode 100644 tendrl/node_agent/alerts/handlers/mount_point_handler.py create mode 100644 tendrl/node_agent/alerts/handlers/swap_handler.py create mode 100644 tendrl/node_agent/objects/service/__init__.py create mode 100644 tendrl/node_agent/objects/service/atoms/__init__.py create mode 100644 tendrl/node_agent/objects/service/atoms/check_service_status.py create mode 100644 tendrl/node_agent/tests/test_alert_manager.py diff --git a/tendrl/node_agent/alerts/__init__.py b/tendrl/node_agent/alerts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tendrl/node_agent/alerts/alert_socket.py b/tendrl/node_agent/alerts/alert_socket.py new file mode 100644 index 00000000..f16cd0e0 --- /dev/null +++ b/tendrl/node_agent/alerts/alert_socket.py @@ -0,0 +1,70 @@ +from _socket import error as _socket_error +import gevent.event +import gevent.greenlet +from gevent.server import StreamServer +from io import BlockingIOError +import json +import logging +from tendrl.common.alert import AlertUtils +from tendrl.common.config import ConfigNotFound +from tendrl.common.config import TendrlConfig +from tendrl.node_agent.alerts.base_alert_handler import AlertHandlerManager +from tendrl.node_agent.alerts.base_alert_handler import NoHandlerException +import tendrl.node_agent.manager.utils as utils +import uuid + +config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") +LOG = logging.getLogger(__name__) +RECEIVE_DATA_SIZE = 4096 + + +class AlertsManager(gevent.greenlet.Greenlet): + + def read_socket(self, sock, address): + try: + data = sock.recv(RECEIVE_DATA_SIZE) + alert_utils = AlertUtils() + alert_json = json.loads(data) + alert_json['alert_id'] = str(uuid.uuid4()) + alert_json['significance'] = 'HIGH' + alert_json['node_id'] = utils.get_local_node_context() + alert_json['ackedby'] = '' + alert_json['acked'] = False + alert = alert_utils.to_obj(alert_json) + AlertHandlerManager().handle(alert) + except ( + KeyError, + TypeError, + ValueError, + NoHandlerException + ) as ex: + LOG.error('Failed to handle data on alert socket.Error %s' % ex) + + def __init__(self): + super(AlertsManager, self).__init__() + try: + self.hostname = config.get( + "node-agent", + "tendrl_alerts_socket_addr" + ) + self.port = config.get( + "node-agent", + "tendrl_alerts_socket_port" + ) + self.server = StreamServer( + (self.hostname, int(self.port)), + self.read_socket + ) + except ConfigNotFound as ex: + LOG.error('Failed to fetch alerting socket configurations.\ + Error %s' % ex) + + def _run(self): + try: + self.server.serve_forever() + except (TypeError, BlockingIOError, _socket_error, ValueError) as ex: + LOG.error('Error trying to serve the alerting socket forever.\ + Error %s' % ex) + + def stop(self): + self.server.close() diff --git a/tendrl/node_agent/alerts/base_alert_handler.py b/tendrl/node_agent/alerts/base_alert_handler.py new file mode 100644 index 00000000..08c2d774 --- /dev/null +++ b/tendrl/node_agent/alerts/base_alert_handler.py @@ -0,0 +1,118 @@ +import datetime +import etcd +import importlib +import inspect +import logging +import os +import six +from tendrl.common.alert import AlertUtils +from tendrl.common.config import TendrlConfig +from tendrl.common.singleton import to_singleton + +config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") +LOG = logging.getLogger(__name__) + + +class NoHandlerException(Exception): + pass + + +class HandlerMount(type): + + def __init__(cls, name, bases, attrs): + if not hasattr(cls, 'handlers'): + cls.handlers = [] + else: + cls.register_handler(cls) + + def register_handler(cls, handler): + instance = handler() + cls.handlers.append(instance) + + +@six.add_metaclass(HandlerMount) +class AlertHandler(object): + def __init__(self): + self.time_stamp = datetime.datetime.now().isoformat() + self.alert = None + self.handles = '' + + def update_alert(self): + # Fetch alerts in etcd + try: + alerts = AlertUtils().get_alerts() + # Check if similar alert already exists + for curr_alert in alerts: + # If similar alert exists, update the similar alert to etcd + if AlertUtils().is_same(self.alert, curr_alert): + self.alert = AlertUtils().update(self.alert, curr_alert) + AlertUtils().store_alert(self.alert) + return + # else add this new alert to etcd + AlertUtils().store_alert(self.alert) + except etcd.EtcdKeyNotFound: + AlertUtils().store_alert(self.alert) + except etcd.EtcdConnectionFailed as ex: + LOG.error( + 'Failed to fetch existing alerts.Error %s' % ex, + exc_info=True + ) + + def handle(self, alert_obj): + try: + self.alert = alert_obj + self.alert.significance = 'HIGH' + self.update_alert() + except Exception as ex: + LOG.error( + 'Failed to handle the alert %s.Error %s' + % (str(alert_obj.to_json_string()), str(ex)), + exc_info=True + ) + + +@to_singleton +class AlertHandlerManager(object): + def load_handlers(self): + try: + path = os.path.dirname(os.path.abspath(__file__)) + '/handlers' + pkg = 'tendrl.node_agent.alerts.handlers' + for py in [f[:-3] for f in os.listdir(path) + if f.endswith('.py') and f != '__init__.py']: + handler_name = '.'.join([pkg, py]) + mod = importlib.import_module(handler_name) + clsmembers = inspect.getmembers(mod, inspect.isclass) + for name, cls in clsmembers: + exec("from %s import %s" % (handler_name, name)) + except (SyntaxError, ValueError, ImportError) as ex: + LOG.error('Failed to load the alert handlers. Error %s' % + ex, exc_info=True) + raise ex + + def __init__(self): + try: + self.load_handlers() + etcd_kwargs = { + 'port': int(config.get("commons", "etcd_port")), + 'host': config.get("commons", "etcd_connection") + } + self.etcd_client = etcd.Client(**etcd_kwargs) + alert_handlers = [] + for handler in AlertHandler.handlers: + alert_handlers.append(handler.handles) + self.etcd_client.write( + '/alerting/alert_types/node_agent', + alert_handlers + ) + except (SyntaxError, ValueError, ImportError) as ex: + raise ex + + def handle(self, alert): + for handler in AlertHandler.handlers: + if handler.handles == alert.resource: + handler.handle(alert) + return + raise NoHandlerException( + 'No alert handler defined for %s and hence cannot handle alert %s' + % (alert['resource'], str(alert)) + ) diff --git a/tendrl/node_agent/alerts/handlers/__init__.py b/tendrl/node_agent/alerts/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tendrl/node_agent/alerts/handlers/cpu_handler.py b/tendrl/node_agent/alerts/handlers/cpu_handler.py new file mode 100644 index 00000000..e4f7a843 --- /dev/null +++ b/tendrl/node_agent/alerts/handlers/cpu_handler.py @@ -0,0 +1,7 @@ +from tendrl.node_agent.alerts.base_alert_handler import AlertHandler + + +class CpuHandler(AlertHandler): + def __init__(self): + AlertHandler.__init__(self) + self.handles = 'cpu' diff --git a/tendrl/node_agent/alerts/handlers/memory_handler.py b/tendrl/node_agent/alerts/handlers/memory_handler.py new file mode 100644 index 00000000..4494e2b2 --- /dev/null +++ b/tendrl/node_agent/alerts/handlers/memory_handler.py @@ -0,0 +1,7 @@ +from tendrl.node_agent.alerts.base_alert_handler import AlertHandler + + +class MemoryHandler(AlertHandler): + def __init__(self): + AlertHandler.__init__(self) + self.handles = 'memory' diff --git a/tendrl/node_agent/alerts/handlers/mount_point_handler.py b/tendrl/node_agent/alerts/handlers/mount_point_handler.py new file mode 100644 index 00000000..c25945c4 --- /dev/null +++ b/tendrl/node_agent/alerts/handlers/mount_point_handler.py @@ -0,0 +1,7 @@ +from tendrl.node_agent.alerts.base_alert_handler import AlertHandler + + +class MountPointHandler(AlertHandler): + def __init__(self): + AlertHandler.__init__(self) + self.handles = 'df' diff --git a/tendrl/node_agent/alerts/handlers/swap_handler.py b/tendrl/node_agent/alerts/handlers/swap_handler.py new file mode 100644 index 00000000..13df19a6 --- /dev/null +++ b/tendrl/node_agent/alerts/handlers/swap_handler.py @@ -0,0 +1,7 @@ +from tendrl.node_agent.alerts.base_alert_handler import AlertHandler + + +class SwapHandler(AlertHandler): + def __init__(self): + AlertHandler.__init__(self) + self.handles = 'swap' diff --git a/tendrl/node_agent/manager/manager.py b/tendrl/node_agent/manager/manager.py index e0b3b4d6..42637a8e 100644 --- a/tendrl/node_agent/manager/manager.py +++ b/tendrl/node_agent/manager/manager.py @@ -13,10 +13,9 @@ from tendrl.commons.etcdobj import etcdobj from tendrl.commons.log import setup_logging from tendrl.commons.manager import manager as common_manager +from tendrl.node_agent.alerts.alert_socket import AlertsManager from tendrl.node_agent.discovery.platform.manager import PlatformManager from tendrl.node_agent.discovery.sds.manager import SDSDiscoveryManager -from tendrl.node_agent.manager.tendrl_definitions_node_agent import data as \ - def_data from tendrl.node_agent.manager import utils from tendrl.node_agent.persistence.cpu import Cpu from tendrl.node_agent.persistence.disk import Disk @@ -28,7 +27,6 @@ from tendrl.node_agent.persistence.platform import Platform from tendrl.node_agent.persistence.service import Service from tendrl.node_agent.persistence.tendrl_context import TendrlContext -from tendrl.node_agent.persistence.tendrl_definitions import TendrlDefinitions config = load_config("node-agent", "/etc/tendrl/node-agent/node-agent.conf.yaml") @@ -123,6 +121,19 @@ def __init__(self, machine_id): self.register_node(machine_id) self.load_and_execute_platform_discovery_plugins() self.load_and_execute_sds_discovery_plugins() + self._alerts_manager = AlertsManager( + config['configuration']['tendrl_alerts_socket_addr'], + config['configuration']['tendrl_alerts_socket_port'], + self.etcd_client + ) + + def start(self): + super(NodeAgentManager, self).start() + self._alerts_manager.start() + + def stop(self): + super(NodeAgentManager, self).stop() + self._alerts_manager.stop() def register_node(self, machine_id): update_node_context(self, machine_id) @@ -143,8 +154,7 @@ def register_node(self, machine_id): status="UP" ) ) - self.persister_thread.update_tendrl_definitions(TendrlDefinitions( - updated=str(time.time()), data=def_data)) + self.persister_thread.append_definitions() def on_pull(self, raw_data): LOG.info("on_pull, Updating Node_context data") diff --git a/tendrl/node_agent/manager/tendrl_definitions_node_agent.py b/tendrl/node_agent/manager/tendrl_definitions_node_agent.py index 756f9272..a64acf7e 100644 --- a/tendrl/node_agent/manager/tendrl_definitions_node_agent.py +++ b/tendrl/node_agent/manager/tendrl_definitions_node_agent.py @@ -318,6 +318,16 @@ run: tendrl.node_agent.atoms.service.configure.Configure type: Update uuid: b90a0d97-8c9f-4ab1-8f64-dbb5638159a3 + check_service_status: + enabled: true + inputs: + mandatory: + - Node.fqdn + - Service.name + name: "check whether the service is running" + run: tendrl.node_agent.objects.service.atoms.check_service_status.CheckServiceStatus + type: Create + uuid: eda0b13a-7362-48d5-b5ca-4b6d6533a5ab attrs: config_data: help: "Configuration data for the service" diff --git a/tendrl/node_agent/objects/node/atoms/cmd.py b/tendrl/node_agent/objects/node/atoms/cmd.py index 73d30001..2c682fae 100644 --- a/tendrl/node_agent/objects/node/atoms/cmd.py +++ b/tendrl/node_agent/objects/node/atoms/cmd.py @@ -1,3 +1,4 @@ +import shlex import subprocess from tendrl.commons.atoms import base_atom @@ -6,6 +7,6 @@ class Cmd(base_atom.BaseAtom): def run(self, parameters): cmd = parameters.get("Node.cmd_str") - cmd = ["nohup"] + cmd.split(" ") + cmd = ["nohup"] + shlex.split(cmd) subprocess.Popen(cmd) return True diff --git a/tendrl/node_agent/objects/service/__init__.py b/tendrl/node_agent/objects/service/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tendrl/node_agent/objects/service/atoms/__init__.py b/tendrl/node_agent/objects/service/atoms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tendrl/node_agent/objects/service/atoms/check_service_status.py b/tendrl/node_agent/objects/service/atoms/check_service_status.py new file mode 100644 index 00000000..0930714b --- /dev/null +++ b/tendrl/node_agent/objects/service/atoms/check_service_status.py @@ -0,0 +1,12 @@ +import os + + +class CheckServiceStatus(object): + def run(self, parameters): + service_name = parameters.get("Service.name") + response = os.system("systemctl status %s" % service_name) + # and then check the response... + if response == 0: + return True + else: + return False diff --git a/tendrl/node_agent/tests/test_alert_manager.py b/tendrl/node_agent/tests/test_alert_manager.py new file mode 100644 index 00000000..e8bcf989 --- /dev/null +++ b/tendrl/node_agent/tests/test_alert_manager.py @@ -0,0 +1,60 @@ +from gevent.server import StreamServer +from mock import MagicMock +import sys +sys.modules['tendrl.commons.config'] = MagicMock() +sys.modules['tendrl.commons.log'] = MagicMock() +from tendrl.node_agent.alerts.alert_socket import AlertsManager +from tendrl.node_agent.alerts.alert_socket import config +del sys.modules['tendrl.commons.log'] +del sys.modules['tendrl.commons.config'] + + +class TestAlertsManager(object): + def test_constructor(self, monkeypatch): + def mock_config(package, parameter): + if parameter == "tendrl_alerts_socket_port": + return '12345' + if parameter == 'tendrl_alerts_socket_addr': + return '0.0.0.0' + + monkeypatch.setattr(config, 'get', mock_config) + manager = AlertsManager() + assert isinstance(manager.hostname, str) + assert isinstance(manager.port, str) + assert isinstance(manager.server, StreamServer) + # cleanup server instance + manager.stop() + + def test_start(self, monkeypatch): + def mock_config(package, parameter): + if parameter == "tendrl_alerts_socket_port": + return '12345' + if parameter == 'tendrl_alerts_socket_addr': + return '0.0.0.0' + + monkeypatch.setattr(config, 'get', mock_config) + manager = AlertsManager() + + def mock_start(): + return + + monkeypatch.setattr(manager, 'start', mock_start) + manager.start() + # cleanup server instance + manager.stop() + + def test_stop(self, monkeypatch): + def mock_config(package, parameter): + if parameter == "tendrl_alerts_socket_port": + return '12345' + if parameter == 'tendrl_alerts_socket_addr': + return '0.0.0.0' + + monkeypatch.setattr(config, 'get', mock_config) + manager = AlertsManager() + + def mock_stop(): + return + + monkeypatch.setattr(manager, 'stop', mock_stop) + manager.stop() From 06483c7e0e965d95f69ebf2ddc585243e5a5da30 Mon Sep 17 00:00:00 2001 From: anmolbabu Date: Fri, 13 Jan 2017 12:31:12 +0530 Subject: [PATCH 2/4] Incorporated comments tendrl-bug-id: Tendrl/node_agent#70 Signed-off-by: anmolbabu --- tendrl/node_agent/alerts/alert_socket.py | 6 +++--- tendrl/node_agent/alerts/base_alert_handler.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tendrl/node_agent/alerts/alert_socket.py b/tendrl/node_agent/alerts/alert_socket.py index f16cd0e0..0d866491 100644 --- a/tendrl/node_agent/alerts/alert_socket.py +++ b/tendrl/node_agent/alerts/alert_socket.py @@ -5,9 +5,9 @@ from io import BlockingIOError import json import logging -from tendrl.common.alert import AlertUtils -from tendrl.common.config import ConfigNotFound -from tendrl.common.config import TendrlConfig +from tendrl.commons.alert import AlertUtils +from tendrl.commons.config import ConfigNotFound +from tendrl.commons.config import TendrlConfig from tendrl.node_agent.alerts.base_alert_handler import AlertHandlerManager from tendrl.node_agent.alerts.base_alert_handler import NoHandlerException import tendrl.node_agent.manager.utils as utils diff --git a/tendrl/node_agent/alerts/base_alert_handler.py b/tendrl/node_agent/alerts/base_alert_handler.py index 08c2d774..f02a288d 100644 --- a/tendrl/node_agent/alerts/base_alert_handler.py +++ b/tendrl/node_agent/alerts/base_alert_handler.py @@ -5,9 +5,9 @@ import logging import os import six -from tendrl.common.alert import AlertUtils -from tendrl.common.config import TendrlConfig -from tendrl.common.singleton import to_singleton +from tendrl.commons.alert import AlertUtils +from tendrl.commons.config import TendrlConfig +from tendrl.commons.singleton import to_singleton config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") LOG = logging.getLogger(__name__) From 0ab981a0ae88c2cbb56ca1b1c66201c987a2ffe3 Mon Sep 17 00:00:00 2001 From: anmolbabu Date: Sat, 14 Jan 2017 17:34:08 +0530 Subject: [PATCH 3/4] JSON to YAML transformations tendrl-bug-id: Tendrl/node_agent#70 Signed-off-by: anmolbabu --- tendrl/node_agent/alerts/alert_socket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tendrl/node_agent/alerts/alert_socket.py b/tendrl/node_agent/alerts/alert_socket.py index 0d866491..627a581f 100644 --- a/tendrl/node_agent/alerts/alert_socket.py +++ b/tendrl/node_agent/alerts/alert_socket.py @@ -3,7 +3,6 @@ import gevent.greenlet from gevent.server import StreamServer from io import BlockingIOError -import json import logging from tendrl.commons.alert import AlertUtils from tendrl.commons.config import ConfigNotFound @@ -12,6 +11,7 @@ from tendrl.node_agent.alerts.base_alert_handler import NoHandlerException import tendrl.node_agent.manager.utils as utils import uuid +import yaml config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") LOG = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def read_socket(self, sock, address): try: data = sock.recv(RECEIVE_DATA_SIZE) alert_utils = AlertUtils() - alert_json = json.loads(data) + alert_json = yaml.safe_load(data) alert_json['alert_id'] = str(uuid.uuid4()) alert_json['significance'] = 'HIGH' alert_json['node_id'] = utils.get_local_node_context() From 38136d7b73480709c59c7809af931434d7b7772c Mon Sep 17 00:00:00 2001 From: anmolbabu Date: Wed, 18 Jan 2017 09:18:20 +0530 Subject: [PATCH 4/4] Fixes tendrl-bug-id: Tendrl/node_agent#70 Signed-off-by: anmolbabu --- tendrl/node_agent/alerts/alert_socket.py | 38 +++++++---------- .../node_agent/alerts/base_alert_handler.py | 41 +++++++++++-------- tendrl/node_agent/manager/manager.py | 2 +- tendrl/node_agent/persistence/persister.py | 39 ++++++++++++++++-- .../persistence/tendrl_definitions.py | 2 +- tendrl/node_agent/tests/test_alert_manager.py | 31 +++----------- tendrl/node_agent/tests/test_manager.py | 2 +- 7 files changed, 82 insertions(+), 73 deletions(-) diff --git a/tendrl/node_agent/alerts/alert_socket.py b/tendrl/node_agent/alerts/alert_socket.py index 627a581f..2327e7df 100644 --- a/tendrl/node_agent/alerts/alert_socket.py +++ b/tendrl/node_agent/alerts/alert_socket.py @@ -3,19 +3,17 @@ import gevent.greenlet from gevent.server import StreamServer from io import BlockingIOError +import json import logging from tendrl.commons.alert import AlertUtils -from tendrl.commons.config import ConfigNotFound -from tendrl.commons.config import TendrlConfig from tendrl.node_agent.alerts.base_alert_handler import AlertHandlerManager from tendrl.node_agent.alerts.base_alert_handler import NoHandlerException import tendrl.node_agent.manager.utils as utils import uuid -import yaml -config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") LOG = logging.getLogger(__name__) RECEIVE_DATA_SIZE = 4096 +db_client = None class AlertsManager(gevent.greenlet.Greenlet): @@ -23,15 +21,15 @@ class AlertsManager(gevent.greenlet.Greenlet): def read_socket(self, sock, address): try: data = sock.recv(RECEIVE_DATA_SIZE) - alert_utils = AlertUtils() - alert_json = yaml.safe_load(data) + alert_utils = AlertUtils(db_client) + alert_json = json.loads(data) alert_json['alert_id'] = str(uuid.uuid4()) alert_json['significance'] = 'HIGH' alert_json['node_id'] = utils.get_local_node_context() alert_json['ackedby'] = '' alert_json['acked'] = False alert = alert_utils.to_obj(alert_json) - AlertHandlerManager().handle(alert) + AlertHandlerManager(db_client).handle(alert) except ( KeyError, TypeError, @@ -40,24 +38,16 @@ def read_socket(self, sock, address): ) as ex: LOG.error('Failed to handle data on alert socket.Error %s' % ex) - def __init__(self): + def __init__(self, alerts_socket_addr, alerts_socket_port, etcd_client): super(AlertsManager, self).__init__() - try: - self.hostname = config.get( - "node-agent", - "tendrl_alerts_socket_addr" - ) - self.port = config.get( - "node-agent", - "tendrl_alerts_socket_port" - ) - self.server = StreamServer( - (self.hostname, int(self.port)), - self.read_socket - ) - except ConfigNotFound as ex: - LOG.error('Failed to fetch alerting socket configurations.\ - Error %s' % ex) + self.hostname = alerts_socket_addr + self.port = alerts_socket_port + global db_client + db_client = etcd_client + self.server = StreamServer( + (self.hostname, int(self.port)), + self.read_socket + ) def _run(self): try: diff --git a/tendrl/node_agent/alerts/base_alert_handler.py b/tendrl/node_agent/alerts/base_alert_handler.py index f02a288d..520a0e89 100644 --- a/tendrl/node_agent/alerts/base_alert_handler.py +++ b/tendrl/node_agent/alerts/base_alert_handler.py @@ -6,10 +6,8 @@ import os import six from tendrl.commons.alert import AlertUtils -from tendrl.commons.config import TendrlConfig from tendrl.commons.singleton import to_singleton -config = TendrlConfig("node-agent", "/etc/tendrl/tendrl.conf") LOG = logging.getLogger(__name__) @@ -37,32 +35,43 @@ def __init__(self): self.alert = None self.handles = '' - def update_alert(self): + def update_alert(self, etcd_client): # Fetch alerts in etcd try: - alerts = AlertUtils().get_alerts() + alerts = AlertUtils(etcd_client).get_alerts() # Check if similar alert already exists for curr_alert in alerts: # If similar alert exists, update the similar alert to etcd - if AlertUtils().is_same(self.alert, curr_alert): - self.alert = AlertUtils().update(self.alert, curr_alert) - AlertUtils().store_alert(self.alert) + if AlertUtils(etcd_client).is_same(self.alert, curr_alert): + self.alert = AlertUtils( + etcd_client + ).update( + self.alert, + curr_alert + ) + if not AlertUtils( + etcd_client + ).equals( + self.alert, + curr_alert + ): + AlertUtils(etcd_client).store_alert(self.alert) return # else add this new alert to etcd - AlertUtils().store_alert(self.alert) + AlertUtils(etcd_client).store_alert(self.alert) except etcd.EtcdKeyNotFound: - AlertUtils().store_alert(self.alert) + AlertUtils(etcd_client).store_alert(self.alert) except etcd.EtcdConnectionFailed as ex: LOG.error( 'Failed to fetch existing alerts.Error %s' % ex, exc_info=True ) - def handle(self, alert_obj): + def handle(self, alert_obj, etcd_client): try: self.alert = alert_obj self.alert.significance = 'HIGH' - self.update_alert() + self.update_alert(etcd_client) except Exception as ex: LOG.error( 'Failed to handle the alert %s.Error %s' @@ -89,14 +98,10 @@ def load_handlers(self): ex, exc_info=True) raise ex - def __init__(self): + def __init__(self, etcd_client): try: self.load_handlers() - etcd_kwargs = { - 'port': int(config.get("commons", "etcd_port")), - 'host': config.get("commons", "etcd_connection") - } - self.etcd_client = etcd.Client(**etcd_kwargs) + self.etcd_client = etcd_client alert_handlers = [] for handler in AlertHandler.handlers: alert_handlers.append(handler.handles) @@ -110,7 +115,7 @@ def __init__(self): def handle(self, alert): for handler in AlertHandler.handlers: if handler.handles == alert.resource: - handler.handle(alert) + handler.handle(alert, self.etcd_client) return raise NoHandlerException( 'No alert handler defined for %s and hence cannot handle alert %s' diff --git a/tendrl/node_agent/manager/manager.py b/tendrl/node_agent/manager/manager.py index 42637a8e..6cdddd38 100644 --- a/tendrl/node_agent/manager/manager.py +++ b/tendrl/node_agent/manager/manager.py @@ -124,7 +124,7 @@ def __init__(self, machine_id): self._alerts_manager = AlertsManager( config['configuration']['tendrl_alerts_socket_addr'], config['configuration']['tendrl_alerts_socket_port'], - self.etcd_client + self.etcd_orm.client ) def start(self): diff --git a/tendrl/node_agent/persistence/persister.py b/tendrl/node_agent/persistence/persister.py index 101d6abd..e0ade514 100644 --- a/tendrl/node_agent/persistence/persister.py +++ b/tendrl/node_agent/persistence/persister.py @@ -1,10 +1,21 @@ +import etcd +from tendrl.commons.etcdobj.etcdobj import Server as etcd_server from tendrl.commons.persistence.etcd_persister import EtcdPersister +from tendrl.node_agent.manager.tendrl_definitions_node_agent import data as \ + def_data +from tendrl.node_agent.persistence.tendrl_definitions import TendrlDefinitions +import time +import yaml class NodeAgentEtcdPersister(EtcdPersister): - def __init__(self, config): - super(NodeAgentEtcdPersister, self).__init__(config) - self._store = self.get_store() + def __init__(self, config, etcd_client): + super(NodeAgentEtcdPersister, self).__init__(etcd_client) + etcd_kwargs = { + 'port': int(config["configuration"]["etcd_port"]), + 'host': config["configuration"]["etcd_connection"] + } + self._store = etcd_server(etcd_kwargs=etcd_kwargs) def update_cpu(self, cpu): self._store.save(cpu) @@ -32,3 +43,25 @@ def update_tendrl_definitions(self, definition): def update_platform(self, platform): self._store.save(platform) + + def append_definitions(self): + try: + defs_path = 'tendrl_definitions_node_agent/data' + defs = yaml.load(self._store.client.read( + defs_path).value.decode("utf-8")) + perf_defs = yaml.load(def_data) + for key in perf_defs: + if key.startswith('namespace.'): + defs[key] = perf_defs[key] + self.update_tendrl_definitions( + TendrlDefinitions(updated=str( + time.time()), + data=yaml.safe_dump(defs)) + ) + except etcd.EtcdKeyNotFound: + self.update_tendrl_definitions( + TendrlDefinitions( + updated=str(time.time()), + data=def_data + ) + ) diff --git a/tendrl/node_agent/persistence/tendrl_definitions.py b/tendrl/node_agent/persistence/tendrl_definitions.py index d9637189..92b2ef79 100644 --- a/tendrl/node_agent/persistence/tendrl_definitions.py +++ b/tendrl/node_agent/persistence/tendrl_definitions.py @@ -7,6 +7,6 @@ class TendrlDefinitions(EtcdObj): """ # TODO(rohan) add the definitions in etcd at startup - __name__ = '/tendrl_definitions_node-agent' + __name__ = '/tendrl_definitions_node_agent' data = fields.StrField("data") diff --git a/tendrl/node_agent/tests/test_alert_manager.py b/tendrl/node_agent/tests/test_alert_manager.py index e8bcf989..cad608ea 100644 --- a/tendrl/node_agent/tests/test_alert_manager.py +++ b/tendrl/node_agent/tests/test_alert_manager.py @@ -4,21 +4,14 @@ sys.modules['tendrl.commons.config'] = MagicMock() sys.modules['tendrl.commons.log'] = MagicMock() from tendrl.node_agent.alerts.alert_socket import AlertsManager -from tendrl.node_agent.alerts.alert_socket import config del sys.modules['tendrl.commons.log'] del sys.modules['tendrl.commons.config'] class TestAlertsManager(object): def test_constructor(self, monkeypatch): - def mock_config(package, parameter): - if parameter == "tendrl_alerts_socket_port": - return '12345' - if parameter == 'tendrl_alerts_socket_addr': - return '0.0.0.0' - - monkeypatch.setattr(config, 'get', mock_config) - manager = AlertsManager() + etcd_client = MagicMock() + manager = AlertsManager('0.0.0.0', '12345', etcd_client) assert isinstance(manager.hostname, str) assert isinstance(manager.port, str) assert isinstance(manager.server, StreamServer) @@ -26,14 +19,8 @@ def mock_config(package, parameter): manager.stop() def test_start(self, monkeypatch): - def mock_config(package, parameter): - if parameter == "tendrl_alerts_socket_port": - return '12345' - if parameter == 'tendrl_alerts_socket_addr': - return '0.0.0.0' - - monkeypatch.setattr(config, 'get', mock_config) - manager = AlertsManager() + etcd_client = MagicMock() + manager = AlertsManager('0.0.0.0', '12345', etcd_client) def mock_start(): return @@ -44,14 +31,8 @@ def mock_start(): manager.stop() def test_stop(self, monkeypatch): - def mock_config(package, parameter): - if parameter == "tendrl_alerts_socket_port": - return '12345' - if parameter == 'tendrl_alerts_socket_addr': - return '0.0.0.0' - - monkeypatch.setattr(config, 'get', mock_config) - manager = AlertsManager() + etcd_client = MagicMock() + manager = AlertsManager('0.0.0.0', '12345', etcd_client) def mock_stop(): return diff --git a/tendrl/node_agent/tests/test_manager.py b/tendrl/node_agent/tests/test_manager.py index 45fd6482..21ab196c 100644 --- a/tendrl/node_agent/tests/test_manager.py +++ b/tendrl/node_agent/tests/test_manager.py @@ -42,7 +42,7 @@ def test_manager_constructor(self): def test_register_node(self): self.manager.persister_thread.update_node_context.assert_called() self.manager.persister_thread.update_node_context.assert_called() - self.manager.persister_thread.update_tendrl_definitions.assert_called() + self.manager.persister_thread.append_definitions.assert_called() class TestNodeAgentSyncStateThread(object):