Avoid redundant port counter collection from virtualized instances, such as Cisco VRF

NOTES.rst

@@ -52,6 +52,27 @@ are:
 .. _`Rittal liquid cooling package (in-row liquid coolers)`: http://www.rittal.com/com-en/product/list.action?categoryPath=/PG0001/PG0168KLIMA1/PGR1951KLIMA1/PG1023KLIMA1
 
 
+Changes to bulk import formats
+------------------------------
+
+The IP Device (Netbox) bulk import format has changed. Two new columns have
+been added, so that the format is now specified as::
+
+    roomid:ip:orgid:catid[:snmp_version:ro:rw:master:function:data:netboxgroup:...]
+
+The new columns are:
+
+snmp_version
+  Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the
+  bulk import format was not updated in the same release, so any device added
+  through the SeedDB bulk import function would default to SNMP v2c. Valid
+  values here are 1 or 2.
+
+master
+  If this device is a virtual instance on another physical device, specify the
+  sysname or IP address of the master in this column. You may have to bulk
+  import multiple times if the master devices are part of the same bulk import
+  file.
 
 NAV 4.6
 ========

bin/dump.py

@@ -60,13 +60,15 @@ class Handlers(object):
     @staticmethod
     def netbox():
         """Outputs a line for each netbox in the database"""
-        header("#roomid:ip:orgid:catid:[ro:rw:function:"
+        header("#roomid:ip:orgid:catid:[snmp_version:ro:rw:function:"
                "key1=value1|key2=value2:"
                "devicegroup1:devicegroup2..]")
         all_functions = manage.NetboxInfo.objects.filter(key='function')
         for box in manage.Netbox.objects.all():
             line = [box.room_id, box.ip, box.organization_id, box.category_id,
-                    box.read_only or "", box.read_write or ""]
+                    str(box.snmp_version) if box.snmp_version else "",
+                    box.read_only or "", box.read_write or "",
+                    box.master.sysname if box.master else ""]
             functions = all_functions.filter(netbox=box)
             functions = str.join(", ", functions)
             line.append(functions)

htdocs/sass/nav/seeddb.scss

@@ -1,5 +1,9 @@
 .readonly {
-		border: none;
+  border: none;
+}
+
+.advanced {
+  display: none;
 }
 
 /* Style the tables on the index page */
@@ -21,9 +25,9 @@
 }
 
 .required:after {
-	content: " *";
-	color: #F00;
-	font-weight: bold;
+        content: " *";
+        color: #F00;
+        font-weight: bold;
 }
 
 /* Style the general info tables */

htdocs/static/js/src/seeddb_netbox.js

@@ -0,0 +1,61 @@
+require(['libs/select2.min'], function() {
+    $(function() {
+        var toggleTrigger = $('.advanced-toggle'),
+            fa = toggleTrigger.find('.fa'),
+            advanced = $('.advanced'),
+            storageKey = 'NAV.seeddb.advanced.show';
+
+        function isHidden(element) {
+            return element.offsetParent === null;
+        }
+
+        function toggle() {
+            advanced.slideToggle(function(){
+                if (isHidden(this)) {
+                    console.log('Setting storagekey to ', 0);
+                    localStorage.setItem(storageKey, '0');
+                } else {
+                    console.log('Setting storagekey to ', 1);
+                    localStorage.setItem(storageKey, '1');
+                }
+            });
+            fa.toggleClass('fa-caret-square-o-right fa-caret-square-o-down');
+        }
+
+        toggleTrigger.on('click', function(event) {
+            event.preventDefault();
+            toggle();
+        });
+
+        // Show element if localstorage says so
+        if (+localStorage.getItem(storageKey) === 1) {
+            advanced.show();
+            fa.toggleClass('fa-caret-square-o-right fa-caret-square-o-down');
+        }
+
+
+        // Master- and instancefield are mutually exclusive. Try to enforce that.
+        var $masterField = $('#id_master').select2();
+        var $instanceField = $('#id_virtual_instance').select2();
+
+        function checkFields() {
+            if ($masterField.val()) {
+                $instanceField.select2('enable', false);
+            } else {
+                $instanceField.select2('enable', true);
+            }
+
+            if ($instanceField.val() && !$masterField[0].disabled) {
+                $masterField.select2('enable', false);
+            } else {
+                $masterField.select2('enable', true);
+            }
+
+
+        }
+
+        $masterField.on('change', checkFields);
+        $instanceField.on('change', checkFields);
+
+    });
+});

python/nav/bulkimport.py

@@ -25,6 +25,7 @@
 from nav.models.manage import Prefix, Vlan, NetType
 from nav.models.cabling import Cabling, Patch
 from nav.models.service import Service, ServiceProperty
+from nav.util import is_valid_ip
 from nav.web.servicecheckers import get_description
 
 from nav.bulkparse import BulkParseError
@@ -94,11 +95,21 @@ def _create_objects_from_row(self, row):
     @staticmethod
     def _get_netbox_from_row(row):
         netbox = Netbox(ip=row['ip'], read_only=row['ro'],
-                        read_write=row['rw'], snmp_version=2)
+                        read_write=row['rw'],
+                        snmp_version=row['snmp_version'] or 2)
         netbox.room = get_object_or_fail(Room, id=row['roomid'])
         netbox.organization = get_object_or_fail(Organization, id=row['orgid'])
         netbox.category = get_object_or_fail(Category, id=row['catid'])
         netbox.sysname = netbox.ip
+
+        master = row.get('master')
+        if master:
+            if is_valid_ip(master, use_socket_lib=True):
+                netbox.master = get_object_or_fail(Netbox, ip=master)
+            else:
+                netbox.master = get_object_or_fail(Netbox,
+                                                   sysname__startswith=master)
+
         return netbox
 
     @staticmethod

python/nav/bulkparse.py

@@ -137,7 +137,8 @@ def next(self):
 
 class NetboxBulkParser(BulkParser):
     """Parses the netbox bulk format"""
-    format = ('roomid', 'ip', 'orgid', 'catid', 'ro', 'rw', 'function', 'data')
+    format = ('roomid', 'ip', 'orgid', 'catid', 'snmp_version', 'ro', 'rw',
+              'master', 'function', 'data')
     required = 4
     restkey = 'netboxgroup'
 
@@ -150,6 +151,15 @@ def _validate_ip(value):
         else:
             return True
 
+    @staticmethod
+    def _validate_snmp_version(value):
+        if not value:
+            return True  # empty values are ok
+        try:
+            return int(value) in (1, 2)
+        except ValueError:
+            return False
+
     @staticmethod
     def _validate_data(datastring):
         try:

python/nav/eventengine/plugins/linkstate.py

@@ -14,6 +14,7 @@
 # along with NAV. If not, see <http://www.gnu.org/licenses/>.
 #
 """"linkState event plugin"""
+import copy
 
 from nav.config import ConfigurationError
 from nav.eventengine.alerts import AlertGenerator
@@ -44,6 +45,11 @@ def get_link_partner(self):
         """Returns the link partner of the target interface"""
         return self.get_target().to_netbox
 
+    def handle(self):
+        if self._is_a_master_for_virtualized_instances():
+            self._copy_event_for_instances()
+        return super(LinkStateHandler, self).handle()
+
     def _handle_end(self):
         self._post_event_if_aggregate_restored()  # always verify aggregates
         return super(LinkStateHandler, self)._handle_end()
@@ -160,6 +166,39 @@ def _get_aggregate_link_event(self, start):
         EventVar(event_queue=event, variable='aggregate_ifalias',
                  value=target.ifalias or '').save()
 
+    #
+    # Methods to handle duplication of events for virtualized netbox instances
+    #
+
+    def _is_a_master_for_virtualized_instances(self):
+        ifc = self.get_target()
+        return ifc and ifc.netbox and ifc.netbox.instances.count() > 0
+
+    def _copy_event_for_instances(self):
+        ifc = self.get_target()
+        netbox = ifc.netbox
+        for instance in netbox.instances.all():
+            self._copy_event_for_instance(netbox, instance, ifc)
+
+    def _copy_event_for_instance(self, netbox, instance, ifc):
+        try:
+            other_ifc = Interface.objects.get(netbox=instance,
+                                              ifname=ifc.ifname)
+        except Interface.DoesNotExist:
+            self._logger.info("interface %s does not exist on instance %s",
+                              ifc.ifname, instance)
+            return
+
+        new_event = copy.copy(self.event)  # type: nav.models.event.EventQueue
+        new_event.pk = None
+        new_event.netbox = instance
+        new_event.device = None
+        new_event.subid = other_ifc.pk
+
+        self._logger.info('duplicating linkState event for %s to %s',
+                          ifc, instance)
+        new_event.save()
+
 
 class LinkStateConfiguration(object):
     """Retrieves configuration options for the LinkStateHandler"""

python/nav/ipdevpoll/__init__.py

@@ -97,3 +97,16 @@ def full_name(self):
         """Return the full module and class name of this instance."""
         return "%s.%s" % (self.__class__.__module__,
                           self.__class__.__name__)
+
+    def _get_netbox_list(self):
+        """Returns a list of netbox names to make metrics for. Will return just
+        the one netbox in most instances, but for situations with multiple
+        virtual device contexts, all the subdevices will be returned.
+
+        """
+        netboxes = [self.netbox.sysname]
+        instances = self.netbox.instances.values_list('sysname', flat=True)
+        netboxes.extend(instances)
+        self._logger.debug("duplicating metrics for these netboxes: %s",
+                           netboxes)
+        return netboxes

python/nav/ipdevpoll/plugins/linkstate.py

@@ -14,24 +14,30 @@
 # License along with NAV. If not, see <http://www.gnu.org/licenses/>.
 #
 """Collects interface link states and dispatches NAV events on changes"""
+from twisted.internet.defer import inlineCallbacks, returnValue
 
 from nav.mibs import reduce_index
 from nav.mibs.if_mib import IfMib
 
-from nav.models.manage import Interface
-
 from nav.ipdevpoll import Plugin
 from nav.ipdevpoll import shadows
 
 
 class LinkState(Plugin):
+    """Monitors interface link states"""
 
+    @inlineCallbacks
     def handle(self):
-        self.ifmib = IfMib(self.agent)
-        df = self.ifmib.retrieve_columns(
-            ['ifName', 'ifAdminStatus', 'ifOperStatus'])
-        df.addCallback(reduce_index)
-        return df.addCallback(self._put_results)
+        if self.netbox.master:
+            self._logger.debug("this is a virtual instance of %s, not polling",
+                               self.netbox.master)
+            returnValue(None)
+
+        ifmib = IfMib(self.agent)
+        result = yield ifmib.retrieve_columns(
+            ['ifName', 'ifAdminStatus', 'ifOperStatus']).addCallback(
+            reduce_index)
+        self._put_results(result)
 
     def _put_results(self, results):
         netbox = self.containers.factory(None, shadows.Netbox)

python/nav/ipdevpoll/plugins/statports.py

@@ -17,6 +17,7 @@
 import time
 from twisted.internet import defer
 from nav.ipdevpoll import Plugin
+from nav.ipdevpoll import db
 from nav.metrics.carbon import send_metrics
 from nav.metrics.templates import metric_path_for_interface
 from nav.mibs import reduce_index
@@ -71,9 +72,15 @@ def can_handle(cls, netbox):
 
     @defer.inlineCallbacks
     def handle(self):
+        if self.netbox.master:
+            yield db.run_in_thread(self._log_instance_details)
+            defer.returnValue(None)
+
         timestamp = time.time()
         stats = yield self._get_stats()
-        tuples = list(self._make_metrics(stats, timestamp))
+        netboxes = yield db.run_in_thread(self._get_netbox_list)
+        tuples = list(self._make_metrics(stats, netboxes=netboxes,
+                                         timestamp=timestamp))
         if tuples:
             self._logger.debug("Counters collected")
             send_metrics(tuples)
@@ -95,7 +102,7 @@ def _get_stats(self):
 
         defer.returnValue(stats)
 
-    def _make_metrics(self, stats, timestamp=None):
+    def _make_metrics(self, stats, netboxes, timestamp=None):
         timestamp = timestamp or time.time()
         hc_counters = False
 
@@ -104,18 +111,31 @@ def _make_metrics(self, stats, timestamp=None):
             for key in LOGGED_COUNTERS:
                 if key not in row:
                     continue
-                path = metric_path_for_interface(
-                    self.netbox, row['ifName'] or row['ifDescr'], key)
                 value = row[key]
                 if value is not None:
-                    yield (path, (timestamp, value))
+                    for netbox in netboxes:
+                        # duplicate metrics for all involved netboxes
+                        path = metric_path_for_interface(
+                            netbox, row['ifName'] or row['ifDescr'], key)
+                        yield (path, (timestamp, value))
 
         if stats:
             if hc_counters:
                 self._logger.debug("High Capacity counters used")
             else:
                 self._logger.debug("High Capacity counters NOT used")
 
+    def _log_instance_details(self):
+        netbox = self.netbox
+
+        my_ifcs = netbox.interface_set.values_list('ifname', flat=True)
+        masters_ifcs = netbox.master.interface_set.values_list('ifname',
+                                                               flat=True)
+        local_ifcs = set(masters_ifcs) - set(my_ifcs)
+
+        self._logger.debug("local interfaces (that do not exist on master "
+                           "%s): %r", self.netbox.master, local_ifcs)
+
 
 def use_hc_counters(row):
     """