Merge 73459ec into 3feb747

tests/plugins/polling/interface/test_plugin_polling_device_interface_metrics.py

@@ -5,6 +5,7 @@
 from tests.plugins.helpers import SNMPPollingPluginTestFramework, setup_module_default, tear_down_module_default
 from yahoo_panoptes.plugins.polling.interface.plugin_polling_device_interface_metrics import \
     PluginPollingDeviceInterfaceMetrics, _MISSING_METRIC_VALUE, _INTERFACE_STATES
+from yahoo_panoptes.plugins.polling.utilities.polling_status import DEVICE_METRICS_STATES
 from yahoo_panoptes.framework.plugins.panoptes_base_plugin import PanoptesPluginRuntimeError
 
 _MOCK_INTERFACE_ENTRY = '0'
@@ -41,10 +42,40 @@ class TestPluginPollingDeviceInterfaceMetrics(SNMPPollingPluginTestFramework, un
         'x509': {'x509_secured_requests': 0}
     }
 
-    plugin_metrics_function = "get_results"
     plugin_class = PluginPollingDeviceInterfaceMetrics
     path = pwd
 
+    def test_inactive_port(self):
+        pass
+
+    def test_no_service_active(self):
+        pass
+
+    # def test_no_service_active(self):
+    #     """Tests a valid resource_endpoint with no service active"""
+    #     self._resource_endpoint = '127.0.0.2'
+    #     self._snmp_conf['timeout'] = self._snmp_failure_timeout
+    #     self.set_panoptes_resource()
+    #     self.set_plugin_context()
+    #
+    #     plugin = self.plugin_class()
+    #
+    #     if self._plugin_conf.get('enrichment'):
+    #         if self._plugin_conf['enrichment'].get('preload'):
+    #             result = plugin.run(self._plugin_context)
+    #             result = self._remove_timestamps(result)
+    #
+    #             self.assertEqual(len(result), 1)
+    #             self.assertEqual(
+    #                 result[0]['metrics'][0]['metric_value'],
+    #                 DEVICE_METRICS_STATES.TIMEOUT
+    #             )
+    #
+    #     self._resource_endpoint = '127.0.0.1'
+    #     self._snmp_conf['timeout'] = self.snmp_timeout
+    #     self.set_panoptes_resource()
+    #     self.set_plugin_context()
+
     def test_missing_interface(self):
         plugin = self.plugin_class()
         plugin.run(self._plugin_context)
@@ -84,37 +115,3 @@ def test_dot3table_stats_exception_handling(self):
         plugin.get_errors_frame = mock_get_errors_frame
         self.assertRaises(Exception, plugin.run(self._plugin_context))
 
-    def test_no_service_active(self):
-        self._resource_endpoint = '127.0.0.2'
-        self._snmp_conf['timeout'] = self._snmp_failure_timeout
-        self.set_panoptes_resource()
-        self.set_plugin_context()
-
-        # Actually get a new instance, else residual metrics may still exist from other tests
-        plugin = self.plugin_class()
-        results = plugin.run(self._plugin_context)
-
-        if self.uses_polling_status is True:
-            self.assertEqual(len(results.metrics_groups), 1)
-        else:
-            self.assertEqual(len(results.metrics_groups), 0)
-
-        self._resource_endpoint = '127.0.0.1'
-        self._snmp_conf['timeout'] = self._snmp_timeout
-        self.set_panoptes_resource()
-        self.set_plugin_context()
-
-    def test_invalid_resource_endpoint(self):
-        self._resource_endpoint = '127.0.0.257'
-        self._snmp_conf['timeout'] = self._snmp_failure_timeout
-        self.set_panoptes_resource()
-        self.set_plugin_context()
-
-        plugin = self.plugin_class()
-        with self.assertRaises(PanoptesPluginRuntimeError):
-            plugin.run(self._plugin_context)
-
-        self._resource_endpoint = '127.0.0.1'
-        self._snmp_conf['timeout'] = self.snmp_timeout
-        self.set_panoptes_resource()
-        self.set_plugin_context()

tests/test_ping.py

@@ -9,6 +9,12 @@
 import json  # why isn't the call below recognized?
 
 from yahoo_panoptes.framework.utilities.ping import *
+from yahoo_panoptes.framework.resources import PanoptesResource, PanoptesContext
+from yahoo_panoptes.plugins.polling.utilities.polling_status import PanoptesPollingStatus, \
+    DEVICE_METRICS_STATES
+from yahoo_panoptes.framework.metrics import PanoptesMetricsGroup
+
+from helpers import get_test_conf_file
 
 # For testing locally on OSX
 TEST_PING_RESPONSE = "ping statistics ---\n" \
@@ -17,6 +23,7 @@
 
 
 class TestPing(unittest.TestCase):
+
     def test_code_output(self):
         # Mock the system calls
         subprocess.check_output = Mock(return_value=TEST_PING_RESPONSE)

yahoo_panoptes/plugins/enrichment/interface/arista/plugin_enrichment_interface_arista.py

@@ -1,7 +1,8 @@
-from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import PluginEnrichmentInterface
+from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import \
+    PluginEnrichmentInterface, InterfaceEnrichment
 
 
-class PluginEnrichmentAristaInterface(PluginEnrichmentInterface):
+class AristaInterfaceEnrichment(InterfaceEnrichment):
     """
     InterfaceEnrichment class for Arista devices.
     """
@@ -33,3 +34,7 @@ def get_parent_interface_media_type(self, index):
 
     def get_parent_interface_port_speed(self, index):
         return self.get_parent_interface_configured_speed(index)
+
+
+class PluginEnrichmentAristaInterface(PluginEnrichmentInterface):
+    interface_enrichment_class = AristaInterfaceEnrichment

yahoo_panoptes/plugins/enrichment/interface/cisco/plugin_enrichment_interface_cisco.py

@@ -1,35 +1,38 @@
-from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import PluginEnrichmentInterface
+from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import \
+    PluginEnrichmentInterface, InterfaceEnrichment
 
 _PORT_SPEED_TABLE = {'Gi': 10 ** 9, 'Te': 10 ** 10}
 
 
-class PluginEnrichmentCiscoInterface(PluginEnrichmentInterface):
+class CiscoInterfaceEnrichment(InterfaceEnrichment):
+
+    def __init__(self, plugin_context, device_resource, interface_enrichment_oids):
+        super(CiscoInterfaceEnrichment, self).__init__(plugin_context, device_resource, interface_enrichment_oids)
+
     """
     InterfaceEnrichment class for Cisco devices.
     """
     def get_port_speed(self, index):
         """
         Gets the port speed enrichment value for the specified interface associated with the provided index
-
         Args:
             index (int): The index to look up the associated interface in self._interface_table
-
         Returns:
             integer: The port speed, based upon the interface name or copied from the configured_speed enrichment in
                      the default case
         """
+        if self._interface_table[index]['interface_name'] is None:
+            self._interface_table[index]['interface_name'] = self.get_interface_name(index)
         for port_speed_indicator in _PORT_SPEED_TABLE.keys():
             if self._interface_table[index]['interface_name'].startswith(port_speed_indicator):
                 return _PORT_SPEED_TABLE[port_speed_indicator]
-        return super(PluginEnrichmentCiscoInterface, self).get_port_speed(index)
+        return super(CiscoInterfaceEnrichment, self).get_port_speed(index)
 
     def get_parent_interface_name(self, index):
         """
         Gets the parent interface name for the interface associated with the provided index
-
         Args:
             index (int): The index used to look up the associated interface in self._interface_table
-
         Returns:
             string: The name of the parent interface, or self._MISSING_VALUE_STRING if the interface has no parent.
                     For Cisco devices, this is everything to the left of the '.' in the interface name, if a '.' is
@@ -41,3 +44,7 @@ def get_parent_interface_name(self, index):
             return parent_interface_name
         else:
             return self._MISSING_VALUE_STRING
+
+
+class PluginEnrichmentCiscoInterface(PluginEnrichmentInterface):
+    interface_enrichment_class = CiscoInterfaceEnrichment

yahoo_panoptes/plugins/enrichment/interface/juniper/plugin_enrichment_interface_juniper.py

@@ -1,9 +1,10 @@
-from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import PluginEnrichmentInterface
+from yahoo_panoptes.plugins.enrichment.interface.plugin_enrichment_interface import \
+    PluginEnrichmentInterface, InterfaceEnrichment
 
 _PORT_SPEED_TABLE = {'ge': 10 ** 9, 'xe': 10 ** 10, 'xle': 4 * 10 ** 10, 'et': 4 * 10 ** 10, 'fte': 4 * 10 ** 10}
 
 
-class PluginEnrichmentJuniperInterface(PluginEnrichmentInterface):
+class JuniperInterfaceEnrichment(InterfaceEnrichment):
     """
     InterfaceEnrichment class for Juniper devices.
     """
@@ -18,13 +19,15 @@ def get_port_speed(self, index):
             integer: The port speed, based upon the interface name or copied from the configured_speed enrichment in
                      the default case
         """
+        if self._interface_table[index]['interface_name'] is None:
+            self._interface_table[index]['interface_name'] = self.get_interface_name(index)
         for port_speed_indicator in _PORT_SPEED_TABLE.keys():
             if self._interface_table[index]['interface_name'].startswith(port_speed_indicator):
                 if port_speed_indicator == 'et':
-                    if self.resource.resource_metadata['model'] in ['QFX5200', 'QFX10000']:
+                    if self._device_resource.resource_metadata['model'] in ['QFX5200', 'QFX10000']:
                         return 10 ** 11
                 return _PORT_SPEED_TABLE[port_speed_indicator]
-        return super(PluginEnrichmentJuniperInterface, self).get_port_speed(index)
+        return super(JuniperInterfaceEnrichment, self).get_port_speed(index)
 
     def get_parent_interface_name(self, index):
         """
@@ -50,3 +53,7 @@ def get_parent_interface_name(self, index):
         else:
             parent_interface_name = self._MISSING_VALUE_STRING
         return parent_interface_name
+
+
+class PluginEnrichmentJuniperInterface(PluginEnrichmentInterface):
+    interface_enrichment_class = JuniperInterfaceEnrichment

yahoo_panoptes/plugins/enrichment/interface/plugin_enrichment_interface.py

@@ -1,37 +1,57 @@
-from yahoo_panoptes.enrichment.enrichment_plugin import PanoptesEnrichmentPlugin
-from yahoo_panoptes.enrichment.schema.interface import PanoptesInterfaceEnrichmentGroup
+import time
+import requests
+
+from yahoo_panoptes.enrichment.enrichment_plugin import PanoptesEnrichmentPlugin, PanoptesEnrichmentPluginError
 from yahoo_panoptes.framework.enrichment import PanoptesEnrichmentSet, PanoptesEnrichmentGroupSet
-from yahoo_panoptes.framework.plugins.base_snmp_plugin import PanoptesSNMPBaseEnrichmentPlugin
 from yahoo_panoptes.framework.utilities.helpers import transform_octet_to_mac
-from yahoo_panoptes.framework.utilities.snmp.mibs.ifTable import getIfTypeDesc, ifDescr, ifType, ifSpeed, ifPhysAddress
+from yahoo_panoptes.framework.utilities.snmp.mibs.ifTable import getIfTypeDesc, ifDescr, ifType, ifSpeed, \
+    ifPhysAddress
 from yahoo_panoptes.framework.utilities.snmp.mibs.ifXTable import ifAlias, ifName, ifHighSpeed
+from yahoo_panoptes.plugins.helpers.snmp_connections import PanoptesSNMPConnectionFactory
+from yahoo_panoptes.enrichment.schema.interface import PanoptesInterfaceEnrichmentGroup
 
 
-class PluginEnrichmentInterface(PanoptesSNMPBaseEnrichmentPlugin, PanoptesEnrichmentPlugin):
+class InterfaceEnrichment(object):
     _MISSING_VALUE_STRING = u'<not set>'
     _MISSING_METRIC_VALUE = -1
-    interface_enrichment_oids = [ifType, ifDescr, ifName, ifAlias, ifHighSpeed, ifSpeed, ifPhysAddress]
 
-    def __init__(self):
-        super(PluginEnrichmentInterface, self).__init__()
-        self._interface_enrichment_group = None
-        self._interface_enrichment_group_set = None
+    def __init__(self, plugin_context, device_resource, interface_enrichment_oids):
+        self._plugin_context = plugin_context
+        self._logger = plugin_context.logger
+        self._plugin_conf = plugin_context.config
+        try:
+            self._execute_frequency = int(self._plugin_conf['main']['execute_frequency'])
+            self._enrichment_ttl = int(self._plugin_conf['main']['enrichment_ttl'])
+            self._snmp_max_repetitions = int(self._plugin_conf['snmp']['max_repetitions'])
+            self._snmp_timeout = int(self._plugin_conf['snmp']['timeout'])
+            self._snmp_retries = int(self._plugin_conf['snmp']['retries'])
+        except Exception as e:
+            raise PanoptesEnrichmentPluginError(
+                    'Either required configurations not specified or not an integer: %s' % repr(e))
+
+        self._device_resource = device_resource
+        self._device_fqdn = device_resource.resource_endpoint
+        self._interface_enrichment_group = PanoptesInterfaceEnrichmentGroup(enrichment_ttl=self._enrichment_ttl,
+                                                                            execute_frequency=self._execute_frequency)
+        self._interface_enrichment_group_set = PanoptesEnrichmentGroupSet(device_resource)
+        self._interface_enrichment_oids = interface_enrichment_oids
+        self._snmp_connection = None
         self._enrichments_map = dict()
         self._interface_table = dict()
+        self._session = requests.Session()
 
     def _build_enrichments_map(self):
         self._enrichments_map = dict()
-        for enrichment_oid in self.interface_enrichment_oids:
-            for varbind in self._snmp_connection.bulk_walk(enrichment_oid,
-                                                           max_repetitions=self.snmp_configuration.max_repetitions):
+        for enrichment_oid in self._interface_enrichment_oids:
+            for varbind in self._snmp_connection.bulk_walk(enrichment_oid, max_repetitions=self._snmp_max_repetitions):
                 self._enrichments_map[varbind.oid + '.' + varbind.index] = varbind.value
 
     def get_interface_name(self, index):
         return self._enrichments_map.get(ifName + '.' + index, self._MISSING_VALUE_STRING)
 
     @property
-    def host(self):
-        return self._host
+    def device_fqdn(self):
+        return self._device_fqdn
 
     def get_parent_interface_name(self, index):
         """
@@ -114,6 +134,7 @@ def get_if_speed(self, index):
 
     def get_configured_speed(self, index):
         high_speed = self._enrichments_map.get(ifHighSpeed + '.' + index)
+        # TODO Should this actually be if 0 <= high_speed < 4294 (i.e. floor(2^32 / 10^6))?
         if high_speed in ['0', '1', None]:
             speed = self.get_if_speed(index)
             return int(speed) if speed != 0 else (int(high_speed) * 1000000 if high_speed is not None
@@ -164,10 +185,14 @@ def _add_parent_interface_enrichments(self):
                 self.get_parent_interface_configured_speed(index)
             self._interface_table[index]['parent_interface_port_speed'] = self.get_parent_interface_port_speed(index)
 
-    def get_results(self):
-        self._interface_enrichment_group = PanoptesInterfaceEnrichmentGroup(enrichment_ttl=self.enrichment_ttl,
-                                                                            execute_frequency=self.execute_frequency)
-        self._interface_enrichment_group_set = PanoptesEnrichmentGroupSet(self.resource)
+    def get_enrichment(self):
+        try:
+            self._snmp_connection = PanoptesSNMPConnectionFactory.get_snmp_connection(
+                plugin_context=self._plugin_context, resource=self._device_resource,
+                timeout=self._snmp_timeout, retries=self._snmp_retries)
+        except Exception as e:
+            raise PanoptesEnrichmentPluginError('Error while creating snmp connection for the device {}: {}'.
+                                                format(self.device_fqdn, repr(e)))
 
         self._build_enrichments_map()
         self._build_interface_table()
@@ -178,11 +203,54 @@ def get_results(self):
                 self._interface_enrichment_group.add_enrichment_set(PanoptesEnrichmentSet(str(index), enrichment_set))
             except Exception as e:
                 self._logger.error('Error while adding enrichment set {} to enrichment group for the device {}: {}'.
-                                   format(str(index), self.host, repr(e)))
+                                   format(str(index), self.device_fqdn, repr(e)))
 
         self._interface_enrichment_group_set.add_enrichment_group(self._interface_enrichment_group)
 
         self._logger.debug('Interface enrichment for device {} PanoptesEnrichmentGroupSet {}'.
-                           format(self.host, self._interface_enrichment_group_set))
+                           format(self.device_fqdn, self._interface_enrichment_group_set))
 
         return self._interface_enrichment_group_set
+
+
+class PluginEnrichmentInterface(PanoptesEnrichmentPlugin):
+    interface_enrichment_class = InterfaceEnrichment
+
+    def run(self, context):
+        """
+        The main entry point to the plugin
+
+        Args:
+            context (PanoptesPluginContext): The Plugin Context passed by the Plugin Agent
+
+        Returns:
+            PanoptesEnrichmentGroupSet: A non-empty resource set
+
+        Raises:
+            PanoptesEnrichmentPluginError: This exception is raised if any part of the metrics process has errors
+        """
+        logger = context.logger
+
+        start_time = time.time()
+
+        device_resource = context.data
+
+        logger.info('Going to poll resource "%s" for interface enrichment' % device_resource.resource_endpoint)
+
+        interface_enrichment_oids = [ifType, ifDescr, ifName, ifAlias, ifHighSpeed, ifSpeed, ifPhysAddress]
+
+        device_polling = self.interface_enrichment_class(plugin_context=context, device_resource=device_resource,
+                                                         interface_enrichment_oids=interface_enrichment_oids)
+
+        device_results = device_polling.get_enrichment()
+
+        end_time = time.time()
+
+        if device_results:
+            logger.info(
+                'Done polling interface enrichment for resource "%s" in %.2f seconds, %s elements' % (
+                    device_resource.resource_endpoint, end_time - start_time, len(device_results)))
+        else:
+            logger.warn('Error polling interface enrichment for resource %s' % device_resource.resource_endpoint)
+
+        return device_results