From 9cea91e229a7d93b348141e032def46848aac737 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Tue, 21 Mar 2017 08:11:33 +0100 Subject: [PATCH 01/31] (Netbox.master): new field to enable a relation to physical masters i.e. in the case of VRF, the physical node is the master. Other modes of operation that are similar to VRF exist (including VDC). --- python/nav/models/manage.py | 4 +++- sql/changes/sc.04.07.0011.sql | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 sql/changes/sc.04.07.0011.sql diff --git a/python/nav/models/manage.py b/python/nav/models/manage.py index 8535a322ea..a4a0363550 100644 --- a/python/nav/models/manage.py +++ b/python/nav/models/manage.py @@ -99,8 +99,10 @@ class Netbox(models.Model): up_since = models.DateTimeField(db_column='upsince', auto_now_add=True) up_to_date = models.BooleanField(db_column='uptodate', default=False) discovered = models.DateTimeField(auto_now_add=True) - deleted_at = models.DateTimeField(blank=True, null=True, default=None) + master = models.ForeignKey('Netbox', db_column='masterid', null=True, + blank=True, default=None, + related_name='instances') data = hstore.DictionaryField() objects = hstore.HStoreManager() diff --git a/sql/changes/sc.04.07.0011.sql b/sql/changes/sc.04.07.0011.sql new file mode 100644 index 0000000000..7cb687750c --- /dev/null +++ b/sql/changes/sc.04.07.0011.sql @@ -0,0 +1,10 @@ +ALTER TABLE netbox + ADD COLUMN masterid INTEGER DEFAULT NULL, + ADD CONSTRAINT netbox_masterid_fkey + FOREIGN KEY (masterid) + REFERENCES netbox(netboxid) + ON DELETE CASCADE ON UPDATE CASCADE; + +COMMENT ON COLUMN netbox.masterid IS + 'In the case of virtual sub-units, this field references the physical master unit'; + From 533cc768755c80c0be82b61a6736cf40c03fed2c Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Tue, 21 Mar 2017 14:04:14 +0100 Subject: [PATCH 02/31] (StatPorts): remove redundant collection in cases of virtualized instances In the case of a master netbox, duplicate all collected port metrics across all instance netboxes when shipping to Graphite. In the case of a virtualized instance, just debug log a list of local ports (that do not appear on the master) and exit. There is _no_ collection of data from local ports at the moment, as this would require a complete restructuring of things (i.e collection from individual ports, rather than full column snmpwalks for the values we want) --- python/nav/ipdevpoll/plugins/statports.py | 43 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/python/nav/ipdevpoll/plugins/statports.py b/python/nav/ipdevpoll/plugins/statports.py index ae0f093987..f8202f28cd 100644 --- a/python/nav/ipdevpoll/plugins/statports.py +++ b/python/nav/ipdevpoll/plugins/statports.py @@ -17,6 +17,7 @@ import time from twisted.internet import defer from nav.ipdevpoll import Plugin +from nav.ipdevpoll import db from nav.metrics.carbon import send_metrics from nav.metrics.templates import metric_path_for_interface from nav.mibs import reduce_index @@ -71,9 +72,15 @@ def can_handle(cls, netbox): @defer.inlineCallbacks def handle(self): + if self.netbox.master: + yield db.run_in_thread(self._log_instance_details) + defer.returnValue(None) + timestamp = time.time() stats = yield self._get_stats() - tuples = list(self._make_metrics(stats, timestamp)) + netboxes = yield db.run_in_thread(self._get_netbox_list) + tuples = list(self._make_metrics(stats, netboxes=netboxes, + timestamp=timestamp)) if tuples: self._logger.debug("Counters collected") send_metrics(tuples) @@ -95,7 +102,7 @@ def _get_stats(self): defer.returnValue(stats) - def _make_metrics(self, stats, timestamp=None): + def _make_metrics(self, stats, netboxes, timestamp=None): timestamp = timestamp or time.time() hc_counters = False @@ -104,11 +111,13 @@ def _make_metrics(self, stats, timestamp=None): for key in LOGGED_COUNTERS: if key not in row: continue - path = metric_path_for_interface( - self.netbox, row['ifName'] or row['ifDescr'], key) value = row[key] if value is not None: - yield (path, (timestamp, value)) + for netbox in netboxes: + # duplicate metrics for all involved netboxes + path = metric_path_for_interface( + netbox, row['ifName'] or row['ifDescr'], key) + yield (path, (timestamp, value)) if stats: if hc_counters: @@ -116,6 +125,30 @@ def _make_metrics(self, stats, timestamp=None): else: self._logger.debug("High Capacity counters NOT used") + def _get_netbox_list(self): + """Returns a list of netbox names to make metrics for. Will return just + the one netbox in most instances, but for situations with multiple + virtual device contexts, all the subdevices will be returned. + + """ + netboxes = [self.netbox.sysname] + instances = self.netbox.instances.values_list('sysname', flat=True) + netboxes.extend(instances) + self._logger.debug("duplicating metrics for these netboxes: %s", + netboxes) + return netboxes + + def _log_instance_details(self): + netbox = self.netbox + + my_ifcs = netbox.interface_set.values_list('ifname', flat=True) + masters_ifcs = netbox.master.interface_set.values_list('ifname', + flat=True) + local_ifcs = set(masters_ifcs) - set(my_ifcs) + + self._logger.debug("local interfaces (that do not exist on master " + "%s): %r", self.netbox.master, local_ifcs) + def use_hc_counters(row): """ From 5cb0f16f5dacb7df07d43cfcf83203678d5d5d9b Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Wed, 22 Mar 2017 13:45:54 +0100 Subject: [PATCH 03/31] (LinkState): modernize plugin to use inlineCallbacks --- python/nav/ipdevpoll/plugins/linkstate.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/nav/ipdevpoll/plugins/linkstate.py b/python/nav/ipdevpoll/plugins/linkstate.py index 7242b87e38..8e437d341a 100644 --- a/python/nav/ipdevpoll/plugins/linkstate.py +++ b/python/nav/ipdevpoll/plugins/linkstate.py @@ -14,24 +14,25 @@ # License along with NAV. If not, see . # """Collects interface link states and dispatches NAV events on changes""" +from twisted.internet.defer import inlineCallbacks, returnValue from nav.mibs import reduce_index from nav.mibs.if_mib import IfMib -from nav.models.manage import Interface - from nav.ipdevpoll import Plugin from nav.ipdevpoll import shadows class LinkState(Plugin): + """Monitors interface link states""" + @inlineCallbacks def handle(self): - self.ifmib = IfMib(self.agent) - df = self.ifmib.retrieve_columns( - ['ifName', 'ifAdminStatus', 'ifOperStatus']) - df.addCallback(reduce_index) - return df.addCallback(self._put_results) + ifmib = IfMib(self.agent) + result = yield ifmib.retrieve_columns( + ['ifName', 'ifAdminStatus', 'ifOperStatus']).addCallback( + reduce_index) + self._put_results(result) def _put_results(self, results): netbox = self.containers.factory(None, shadows.Netbox) From 6e5ace155ff7cc81a287d4e463b9cf27d2fb0aa7 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 23 Mar 2017 10:45:58 +0100 Subject: [PATCH 04/31] do not poll interface status for virtualized instances. instead, have the eventengine make copies of linkState events for master devices. --- python/nav/eventengine/plugins/linkstate.py | 39 +++++++++++++++++++++ python/nav/ipdevpoll/plugins/linkstate.py | 5 +++ 2 files changed, 44 insertions(+) diff --git a/python/nav/eventengine/plugins/linkstate.py b/python/nav/eventengine/plugins/linkstate.py index 3201bd725a..3b4eb9f515 100644 --- a/python/nav/eventengine/plugins/linkstate.py +++ b/python/nav/eventengine/plugins/linkstate.py @@ -14,6 +14,7 @@ # along with NAV. If not, see . # """"linkState event plugin""" +import copy from nav.config import ConfigurationError from nav.eventengine.alerts import AlertGenerator @@ -44,6 +45,11 @@ def get_link_partner(self): """Returns the link partner of the target interface""" return self.get_target().to_netbox + def handle(self): + if self._is_a_master_for_virtualized_instances(): + self._copy_event_for_instances() + return super(LinkStateHandler, self).handle() + def _handle_end(self): self._post_event_if_aggregate_restored() # always verify aggregates return super(LinkStateHandler, self)._handle_end() @@ -160,6 +166,39 @@ def _get_aggregate_link_event(self, start): EventVar(event_queue=event, variable='aggregate_ifalias', value=target.ifalias or '').save() + # + # Methods to handle duplication of events for virtualized netbox instances + # + + def _is_a_master_for_virtualized_instances(self): + ifc = self.get_target() + return ifc and ifc.netbox and ifc.netbox.instances.count() > 0 + + def _copy_event_for_instances(self): + ifc = self.get_target() + netbox = ifc.netbox + for instance in netbox.instances.all(): + self._copy_event_for_instance(netbox, instance, ifc) + + def _copy_event_for_instance(self, netbox, instance, ifc): + try: + other_ifc = Interface.objects.get(netbox=instance, + ifname=ifc.ifname) + except Interface.DoesNotExist: + self._logger.info("interface %s does not exist on instance %s", + ifc.ifname, instance) + return + + new_event = copy.copy(self.event) # type: nav.models.event.EventQueue + new_event.pk = None + new_event.netbox = instance + new_event.device = None + new_event.subid = other_ifc.pk + + self._logger.info('duplicating linkState event for %s to %s', + ifc, instance) + new_event.save() + class LinkStateConfiguration(object): """Retrieves configuration options for the LinkStateHandler""" diff --git a/python/nav/ipdevpoll/plugins/linkstate.py b/python/nav/ipdevpoll/plugins/linkstate.py index 8e437d341a..7e7b21c2d9 100644 --- a/python/nav/ipdevpoll/plugins/linkstate.py +++ b/python/nav/ipdevpoll/plugins/linkstate.py @@ -28,6 +28,11 @@ class LinkState(Plugin): @inlineCallbacks def handle(self): + if self.netbox.master: + self._logger.debug("this is a virtual instance of %s, not polling", + self.netbox.master) + returnValue(None) + ifmib = IfMib(self.agent) result = yield ifmib.retrieve_columns( ['ifName', 'ifAdminStatus', 'ifOperStatus']).addCallback( From 143926312c296b9a271f8dddc83f0d6b72148ce8 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 09:49:16 +0100 Subject: [PATCH 05/31] (ipdevinfo): display related virtual instances/masters --- templates/ipdevinfo/frag-ipdevinfo.html | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/templates/ipdevinfo/frag-ipdevinfo.html b/templates/ipdevinfo/frag-ipdevinfo.html index fd1b642584..18b8be82d7 100644 --- a/templates/ipdevinfo/frag-ipdevinfo.html +++ b/templates/ipdevinfo/frag-ipdevinfo.html @@ -21,6 +21,13 @@ {{ netbox.sysname }} + {% if netbox.master %} + + Virtual instance of + {{ netbox.master }} + + {% endif %} + Type @@ -194,6 +201,23 @@ {{ netbox.get_gwports.count }} + {% if netbox.instances.count > 0 %} + + Virtual instances + + + + + {% endif %} + From 12b86b01e62ce609495039f8390ab360a7e0e13c Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 10:25:18 +0100 Subject: [PATCH 06/31] add snmp_version column to netbox bulk import format. --- NOTES.rst | 9 +++++++++ bin/dump.py | 3 ++- python/nav/bulkimport.py | 3 ++- python/nav/bulkparse.py | 12 +++++++++++- tests/integration/bulkimport_test.py | 15 ++++++++------- tests/unittests/general/bulkparse_test.py | 8 ++++---- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/NOTES.rst b/NOTES.rst index eaea8b4d53..817485bbd2 100644 --- a/NOTES.rst +++ b/NOTES.rst @@ -52,6 +52,15 @@ are: .. _`Rittal liquid cooling package (in-row liquid coolers)`: http://www.rittal.com/com-en/product/list.action?categoryPath=/PG0001/PG0168KLIMA1/PGR1951KLIMA1/PG1023KLIMA1 +Changes to bulk import formats +------------------------------ + +The IP Device (Netbox) bulk import format has changed. A new column for snmp +version has been added, just before the SNMP read only community column. +Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the +bulk import format was not updated in the same release, so any device added +through the SeedDB bulk import function would default to SNMP v2c. + NAV 4.6 ======== diff --git a/bin/dump.py b/bin/dump.py index f36ef217d1..cf5fb10981 100755 --- a/bin/dump.py +++ b/bin/dump.py @@ -60,12 +60,13 @@ class Handlers(object): @staticmethod def netbox(): """Outputs a line for each netbox in the database""" - header("#roomid:ip:orgid:catid:[ro:rw:function:" + header("#roomid:ip:orgid:catid:[snmp_version:ro:rw:function:" "key1=value1|key2=value2:" "devicegroup1:devicegroup2..]") all_functions = manage.NetboxInfo.objects.filter(key='function') for box in manage.Netbox.objects.all(): line = [box.room_id, box.ip, box.organization_id, box.category_id, + str(box.snmp_version) if box.snmp_version else "", box.read_only or "", box.read_write or ""] functions = all_functions.filter(netbox=box) functions = str.join(", ", functions) diff --git a/python/nav/bulkimport.py b/python/nav/bulkimport.py index db98573c89..3efb7d0dd0 100644 --- a/python/nav/bulkimport.py +++ b/python/nav/bulkimport.py @@ -94,7 +94,8 @@ def _create_objects_from_row(self, row): @staticmethod def _get_netbox_from_row(row): netbox = Netbox(ip=row['ip'], read_only=row['ro'], - read_write=row['rw'], snmp_version=2) + read_write=row['rw'], + snmp_version=row['snmp_version'] or 2) netbox.room = get_object_or_fail(Room, id=row['roomid']) netbox.organization = get_object_or_fail(Organization, id=row['orgid']) netbox.category = get_object_or_fail(Category, id=row['catid']) diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index d7f82a0877..e05cabf5df 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -137,7 +137,8 @@ def next(self): class NetboxBulkParser(BulkParser): """Parses the netbox bulk format""" - format = ('roomid', 'ip', 'orgid', 'catid', 'ro', 'rw', 'function', 'data') + format = ('roomid', 'ip', 'orgid', 'catid', 'snmp_version', 'ro', 'rw', + 'function', 'data') required = 4 restkey = 'netboxgroup' @@ -150,6 +151,15 @@ def _validate_ip(value): else: return True + @staticmethod + def _validate_snmp_version(value): + if not value: + return True # empty values are ok + try: + return int(value) in (1,2) + except ValueError: + return False + @staticmethod def _validate_data(datastring): try: diff --git a/tests/integration/bulkimport_test.py b/tests/integration/bulkimport_test.py index cae6185316..c42dd1946b 100644 --- a/tests/integration/bulkimport_test.py +++ b/tests/integration/bulkimport_test.py @@ -21,7 +21,7 @@ def test_is_generator(self): class TestNetboxImporter(DjangoTransactionTestCase): def test_simple_import_yields_netbox_and_device_model(self): - data = 'myroom:10.0.90.252:myorg:SW:public::' + data = 'myroom:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -31,7 +31,7 @@ def test_simple_import_yields_netbox_and_device_model(self): self.assertTrue(isinstance(objects[0], manage.Netbox), objects[0]) def test_simple_import_yields_objects_with_proper_values(self): - data = 'myroom:10.0.90.252:myorg:SW:public::' + data = 'myroom:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -41,17 +41,18 @@ def test_simple_import_yields_objects_with_proper_values(self): self.assertEquals(netbox.room_id, 'myroom') self.assertEquals(netbox.organization_id, 'myorg') self.assertEquals(netbox.category_id, 'SW') + self.assertEquals(netbox.snmp_version, '1') self.assertEquals(netbox.read_only, 'public') def test_invalid_room_gives_error(self): - data = 'invalid:10.0.90.252:myorg:SW:public::' + data = 'invalid:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() self.assertTrue(isinstance(objects, DoesNotExist)) def test_netbox_function_is_set(self): - data = 'myroom:10.0.90.252:myorg:SW:public::does things:' + data = 'myroom:10.0.90.252:myorg:SW:1:public::does things:' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -69,7 +70,7 @@ def test_get_netboxinfo_from_function(self): self.assertEquals(netboxinfo.value, 'hella') def test_netbox_groups_are_set(self): - data = 'myroom:10.0.90.10:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -101,7 +102,7 @@ def test_duplicate_locations_should_give_error(self): snmp_version=1) netbox.save() - data = 'myroom:10.1.0.1:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.1.0.1:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -109,7 +110,7 @@ def test_duplicate_locations_should_give_error(self): self.assertTrue(isinstance(objects, AlreadyExists)) def test_created_objects_can_be_saved(self): - data = 'myroom:10.0.90.10:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() diff --git a/tests/unittests/general/bulkparse_test.py b/tests/unittests/general/bulkparse_test.py index d72d353e17..8bf50e3c03 100644 --- a/tests/unittests/general/bulkparse_test.py +++ b/tests/unittests/general/bulkparse_test.py @@ -43,7 +43,7 @@ def test_parse_single_line_should_yield_value(self): self.assertTrue(out_data is not None) def test_parse_single_line_yields_columns(self): - data = ("room1:10.0.0.186:myorg:SW:public:secret:doesthings:" + data = ("room1:10.0.0.186:myorg:SW:1:public:secret:doesthings:" "key=value:blah1:blah2") b = NetboxBulkParser(data) out_data = b.next() @@ -59,10 +59,10 @@ def test_get_header(self): self.assertEquals( NetboxBulkParser.get_header(), "#roomid:ip:orgid:catid" - "[:ro:rw:function:data:netboxgroup:...]") + "[:snmp_version:ro:rw:function:data:netboxgroup:...]") def test_two_rows_returned_with_empty_lines_in_input(self): - data = ("room1:10.0.0.186:myorg:SW:public:parrot::\n" + data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" "\n" "room1:10.0.0.187:myorg:OTHER::parrot::\n") b = NetboxBulkParser(data) @@ -70,7 +70,7 @@ def test_two_rows_returned_with_empty_lines_in_input(self): self.assertEquals(len(out_data), 2) def test_three_lines_with_two_rows_should_be_counted_as_three(self): - data = ("room1:10.0.0.186:myorg:SW:public:parrot::\n" + data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" "\n" "room1:10.0.0.187:myorg:OTHER::parrot::\n") b = NetboxBulkParser(data) From 0365c9f15c320a501017d7033e8d9ef8385b6160 Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 10:32:59 +0100 Subject: [PATCH 07/31] Add fields for choosing master and virtual instances on an IP Device --- python/nav/web/seeddb/page/netbox/forms.py | 73 +++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index 78f05b8094..fb5fa85dba 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -18,6 +18,7 @@ from socket import error as SocketError from django import forms +from django.db.models import Q from django_hstore.forms import DictionaryField from crispy_forms.helper import FormHelper from crispy_forms_foundation.layout import (Layout, Row, Column, Submit, @@ -33,6 +34,16 @@ _logger = logging.getLogger(__name__) +class MyModelMultipleChoiceField(forms.ModelMultipleChoiceField): + def __init__(self, queryset, cache_choices=False, required=True, + widget=None, label=None, initial=None, help_text='', *args, + **kwargs): + super(MyModelMultipleChoiceField, self).__init__( + queryset, cache_choices, required, widget, label, initial, + help_text, *args, **kwargs) + self.help_text = help_text + + class NetboxModelForm(forms.ModelForm): """Modelform for netbox for use in SeedDB""" ip = forms.CharField() @@ -42,18 +53,36 @@ class NetboxModelForm(forms.ModelForm): sysname = forms.CharField(required=False) snmp_version = forms.ChoiceField(choices=[('1', '1'), ('2', '2c')], widget=forms.RadioSelect, initial='2') + virtual_instance = MyModelMultipleChoiceField( + queryset=Netbox.objects.none(), required=False, + help_text='The list of virtual instances you are master to') class Meta(object): model = Netbox fields = ['ip', 'room', 'category', 'organization', 'read_only', 'read_write', 'snmp_version', - 'groups', 'sysname', 'type', 'data'] + 'groups', 'sysname', 'type', 'data', 'master', + 'virtual_instance'] + help_texts = { + 'master': 'Set the virtual master of this IP Device' + } def __init__(self, *args, **kwargs): super(NetboxModelForm, self).__init__(*args, **kwargs) self.fields['organization'].choices = create_hierarchy(Organization) + # Master and instance related queries + masters = [n.master.pk for n in + Netbox.objects.filter(master__isnull=False)] + self.fields['master'].queryset = self.create_master_query(masters) + self.fields['virtual_instance'].queryset = self.create_instance_query(masters) + if self.instance.pk: + # Set instances that we are master to as initial values + self.initial['virtual_instance'] = Netbox.objects.filter( + master=self.instance) + if self.instance.pk: + # Set the inital value of the function field try: netboxinfo = self.instance.info_set.get(variable='function') except NetboxInfo.DoesNotExist: @@ -95,12 +124,43 @@ def __init__(self, *args, **kwargs): Fieldset('Meta information', 'function', Field('groups', css_class='select2'), - 'data'), + 'data', + 'master', 'virtual_instance' + ), css_class=css_class), ), Submit('save_ip_device', 'Save IP device') ) + def create_instance_query(self, masters): + """Creates query for virtual instance multiselect""" + if self.instance.master: + # If we have a master, we should not be able to master instances + queryset = Netbox.objects.none() + else: + # - Should not see other masters + # - Should see those we are master for + # - Should see those who have no master + queryset = Netbox.objects.exclude(pk__in=masters).filter( + Q(master=self.instance.pk) | Q(master__isnull=True)) + + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) + + return queryset + + def create_master_query(self, masters): + """Creates query for master dropdown list""" + if self.instance and self.instance.pk in masters: + queryset = Netbox.objects.none() + else: + # - Should not set those who have master as master + queryset = Netbox.objects.filter(master__isnull=True) + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) + + return queryset + def clean_ip(self): """Make sure IP-address is valid""" name = self.cleaned_data['ip'].strip() @@ -156,6 +216,15 @@ def _check_existing_ip(self, ip): if len(msg) > 0: raise IPExistsException(msg) + def save(self, commit=True): + netbox = super(NetboxModelForm, self).save(commit) + instances = self.cleaned_data.get('virtual_instance') + for instance in instances: + instance.master = netbox + instance.save() + + return netbox + class NetboxFilterForm(forms.Form): """Form for filtering netboxes on the list page""" From 6f4625bfc280b7dc466c57bc08a06cf6de03d233 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 10:54:59 +0100 Subject: [PATCH 08/31] pylint violation fix. --- python/nav/bulkparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index e05cabf5df..3b2e2e55f6 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -156,7 +156,7 @@ def _validate_snmp_version(value): if not value: return True # empty values are ok try: - return int(value) in (1,2) + return int(value) in (1, 2) except ValueError: return False From fc9f933854183e9d8c956a12678ac8a6e578e07e Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 11:36:33 +0100 Subject: [PATCH 09/31] add a new master column to the netbox bulk import format. tests and doc updated. --- NOTES.rst | 22 +++++++++++++++++----- bin/dump.py | 3 ++- python/nav/bulkimport.py | 10 ++++++++++ python/nav/bulkparse.py | 2 +- tests/integration/bulkimport_test.py | 13 ++++++++++--- tests/unittests/general/bulkparse_test.py | 8 +++++--- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/NOTES.rst b/NOTES.rst index 817485bbd2..5c6c9180cf 100644 --- a/NOTES.rst +++ b/NOTES.rst @@ -55,12 +55,24 @@ are: Changes to bulk import formats ------------------------------ -The IP Device (Netbox) bulk import format has changed. A new column for snmp -version has been added, just before the SNMP read only community column. -Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the -bulk import format was not updated in the same release, so any device added -through the SeedDB bulk import function would default to SNMP v2c. +The IP Device (Netbox) bulk import format has changed. Two new columns have +been added, so that the format is now specified as:: + roomid:ip:orgid:catid[:snmp_version:ro:rw:master:function:data:netboxgroup:...] + +The new columns are: + +snmp_version + Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the + bulk import format was not updated in the same release, so any device added + through the SeedDB bulk import function would default to SNMP v2c. Valid + values here are 1 or 2. + +master + If this device is a virtual instance on another physical device, specify the + sysname or IP address of the master in this column. You may have to bulk + import multiple times if the master devices are part of the same bulk import + file. NAV 4.6 ======== diff --git a/bin/dump.py b/bin/dump.py index cf5fb10981..d5dd4abd75 100755 --- a/bin/dump.py +++ b/bin/dump.py @@ -67,7 +67,8 @@ def netbox(): for box in manage.Netbox.objects.all(): line = [box.room_id, box.ip, box.organization_id, box.category_id, str(box.snmp_version) if box.snmp_version else "", - box.read_only or "", box.read_write or ""] + box.read_only or "", box.read_write or "", + box.master.sysname if box.master else ""] functions = all_functions.filter(netbox=box) functions = str.join(", ", functions) line.append(functions) diff --git a/python/nav/bulkimport.py b/python/nav/bulkimport.py index 3efb7d0dd0..772c7050d4 100644 --- a/python/nav/bulkimport.py +++ b/python/nav/bulkimport.py @@ -25,6 +25,7 @@ from nav.models.manage import Prefix, Vlan, NetType from nav.models.cabling import Cabling, Patch from nav.models.service import Service, ServiceProperty +from nav.util import is_valid_ip from nav.web.servicecheckers import get_description from nav.bulkparse import BulkParseError @@ -100,6 +101,15 @@ def _get_netbox_from_row(row): netbox.organization = get_object_or_fail(Organization, id=row['orgid']) netbox.category = get_object_or_fail(Category, id=row['catid']) netbox.sysname = netbox.ip + + master = row.get('master') + if master: + if is_valid_ip(master, use_socket_lib=True): + netbox.master = get_object_or_fail(Netbox, ip=master) + else: + netbox.master = get_object_or_fail(Netbox, + sysname__startswith=master) + return netbox @staticmethod diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index 3b2e2e55f6..2eb236bbc2 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -138,7 +138,7 @@ def next(self): class NetboxBulkParser(BulkParser): """Parses the netbox bulk format""" format = ('roomid', 'ip', 'orgid', 'catid', 'snmp_version', 'ro', 'rw', - 'function', 'data') + 'master', 'function', 'data') required = 4 restkey = 'netboxgroup' diff --git a/tests/integration/bulkimport_test.py b/tests/integration/bulkimport_test.py index c42dd1946b..7096176397 100644 --- a/tests/integration/bulkimport_test.py +++ b/tests/integration/bulkimport_test.py @@ -52,7 +52,7 @@ def test_invalid_room_gives_error(self): self.assertTrue(isinstance(objects, DoesNotExist)) def test_netbox_function_is_set(self): - data = 'myroom:10.0.90.252:myorg:SW:1:public::does things:' + data = 'myroom:10.0.90.252:myorg:SW:1:public:::does things:' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -70,7 +70,7 @@ def test_get_netboxinfo_from_function(self): self.assertEquals(netboxinfo.value, 'hella') def test_netbox_groups_are_set(self): - data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV:::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -102,7 +102,7 @@ def test_duplicate_locations_should_give_error(self): snmp_version=1) netbox.save() - data = 'myroom:10.1.0.1:myorg:SRV::::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.1.0.1:myorg:SRV:::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -120,6 +120,13 @@ def test_created_objects_can_be_saved(self): print(repr(obj)) obj.save() + def test_invalid_master_should_give_error(self): + data = 'myroom:10.0.90.10:myorg:SW::::badmaster:functionality' + parser = NetboxBulkParser(data) + importer = NetboxImporter(parser) + _line_num, objects = importer.next() + self.assertTrue(isinstance(objects, DoesNotExist)) + class TestLocationImporter(DjangoTransactionTestCase): def test_import(self): diff --git a/tests/unittests/general/bulkparse_test.py b/tests/unittests/general/bulkparse_test.py index 8bf50e3c03..0afcb90428 100644 --- a/tests/unittests/general/bulkparse_test.py +++ b/tests/unittests/general/bulkparse_test.py @@ -23,13 +23,14 @@ def _validate_one(self, value): b = TestParser(data) try: list(b) - except InvalidFieldValue, error: + except InvalidFieldValue as error: self.assertEquals(error.line_num, 2) self.assertEquals(error.field, 'one') self.assertEquals(error.value, 'once') else: self.fail("No exception raised") + class TestNetboxBulkParser(TestCase): def test_parse_returns_iterator(self): data = "room1:10.0.0.186:myorg:OTHER::parrot::" @@ -43,7 +44,7 @@ def test_parse_single_line_should_yield_value(self): self.assertTrue(out_data is not None) def test_parse_single_line_yields_columns(self): - data = ("room1:10.0.0.186:myorg:SW:1:public:secret:doesthings:" + data = ("room1:10.0.0.186:myorg:SW:1:public:secret:amaster:doesthings:" "key=value:blah1:blah2") b = NetboxBulkParser(data) out_data = b.next() @@ -52,6 +53,7 @@ def test_parse_single_line_yields_columns(self): self.assertEquals(out_data['ip'], '10.0.0.186') self.assertEquals(out_data['orgid'], 'myorg') self.assertEquals(out_data['catid'], 'SW') + self.assertEquals(out_data['master'], 'amaster') self.assertEquals(out_data['data'], 'key=value') self.assertEquals(out_data['netboxgroup'], ['blah1', 'blah2']) @@ -59,7 +61,7 @@ def test_get_header(self): self.assertEquals( NetboxBulkParser.get_header(), "#roomid:ip:orgid:catid" - "[:snmp_version:ro:rw:function:data:netboxgroup:...]") + "[:snmp_version:ro:rw:master:function:data:netboxgroup:...]") def test_two_rows_returned_with_empty_lines_in_input(self): data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" From 53a5d3567ee6dd59973245c01bec95506c66c8f3 Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 13:36:26 +0100 Subject: [PATCH 10/31] Hide advanced options --- htdocs/sass/nav/seeddb.scss | 12 +++++--- python/nav/web/seeddb/page/netbox/forms.py | 10 ++++-- templates/seeddb/netbox_wizard.html | 36 ++++++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/htdocs/sass/nav/seeddb.scss b/htdocs/sass/nav/seeddb.scss index 032a368e30..4c7f9a7ba4 100644 --- a/htdocs/sass/nav/seeddb.scss +++ b/htdocs/sass/nav/seeddb.scss @@ -1,5 +1,9 @@ .readonly { - border: none; + border: none; +} + +.advanced { + display: none; } /* Style the tables on the index page */ @@ -21,9 +25,9 @@ } .required:after { - content: " *"; - color: #F00; - font-weight: bold; + content: " *"; + color: #F00; + font-weight: bold; } /* Style the general info tables */ diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index fb5fa85dba..ab3d70a239 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -22,7 +22,7 @@ from django_hstore.forms import DictionaryField from crispy_forms.helper import FormHelper from crispy_forms_foundation.layout import (Layout, Row, Column, Submit, - Fieldset, Field, Div) + Fieldset, Field, Div, HTML) from nav.web.crispyforms import LabelSubmit, NavButton from nav.models.manage import Room, Category, Organization, Netbox @@ -125,8 +125,12 @@ def __init__(self, *args, **kwargs): 'function', Field('groups', css_class='select2'), 'data', - 'master', 'virtual_instance' - ), + HTML(" Advanced options"), + Div( + 'master', 'virtual_instance', + css_class='advanced' + ) + ), css_class=css_class), ), Submit('save_ip_device', 'Save IP device') diff --git a/templates/seeddb/netbox_wizard.html b/templates/seeddb/netbox_wizard.html index 4bc7dee654..390f0daefc 100644 --- a/templates/seeddb/netbox_wizard.html +++ b/templates/seeddb/netbox_wizard.html @@ -7,6 +7,42 @@ NAV.urls.seeddb = NAV.urls.seeddb || {}; NAV.urls.get_readonly = "{% url 'seeddb-netbox-get-readonly' %}"; NAV.urls.seeddb.verifyAddress = "{% url 'seeddb-netbox-get-address-info' %}"; + + +{% endblock %} + +{% block footer_scripts %} + {% endblock %} From d5f3779f5e572cf6c54583bcf3f7b59174db7f83 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 15:03:46 +0100 Subject: [PATCH 11/31] improve master/instance help texts slightly --- python/nav/web/seeddb/page/netbox/forms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index ab3d70a239..b8515f2b92 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -55,7 +55,7 @@ class NetboxModelForm(forms.ModelForm): widget=forms.RadioSelect, initial='2') virtual_instance = MyModelMultipleChoiceField( queryset=Netbox.objects.none(), required=False, - help_text='The list of virtual instances you are master to') + help_text='The list of virtual instances inside this master device') class Meta(object): model = Netbox @@ -64,7 +64,8 @@ class Meta(object): 'groups', 'sysname', 'type', 'data', 'master', 'virtual_instance'] help_texts = { - 'master': 'Set the virtual master of this IP Device' + 'master': 'Select a master device when this IP Device is a virtual' + ' instance' } def __init__(self, *args, **kwargs): From c09ce44b4825651e08b9f543c9f0dd61d48c4539 Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 15:22:46 +0100 Subject: [PATCH 12/31] Disable mutually exclusive fields --- python/nav/web/seeddb/page/netbox/forms.py | 32 +++++++++------------- templates/seeddb/netbox_wizard.html | 30 +++++++++++++++++++- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index ab3d70a239..ffbeced544 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -127,6 +127,7 @@ def __init__(self, *args, **kwargs): 'data', HTML(" Advanced options"), Div( + HTML('NB: An IP Device cannot both have a master and have virtual instances'), 'master', 'virtual_instance', css_class='advanced' ) @@ -138,30 +139,23 @@ def __init__(self, *args, **kwargs): def create_instance_query(self, masters): """Creates query for virtual instance multiselect""" - if self.instance.master: - # If we have a master, we should not be able to master instances - queryset = Netbox.objects.none() - else: - # - Should not see other masters - # - Should see those we are master for - # - Should see those who have no master - queryset = Netbox.objects.exclude(pk__in=masters).filter( - Q(master=self.instance.pk) | Q(master__isnull=True)) - - if self.instance.pk: - queryset = queryset.exclude(pk=self.instance.pk) + # - Should not see other masters + # - Should see those we are master for + # - Should see those who have no master + queryset = Netbox.objects.exclude(pk__in=masters).filter( + Q(master=self.instance.pk) | Q(master__isnull=True)) + + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) return queryset def create_master_query(self, masters): """Creates query for master dropdown list""" - if self.instance and self.instance.pk in masters: - queryset = Netbox.objects.none() - else: - # - Should not set those who have master as master - queryset = Netbox.objects.filter(master__isnull=True) - if self.instance.pk: - queryset = queryset.exclude(pk=self.instance.pk) + # - Should not set those who have master as master + queryset = Netbox.objects.filter(master__isnull=True) + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) return queryset diff --git a/templates/seeddb/netbox_wizard.html b/templates/seeddb/netbox_wizard.html index 390f0daefc..f70f975e76 100644 --- a/templates/seeddb/netbox_wizard.html +++ b/templates/seeddb/netbox_wizard.html @@ -25,8 +25,10 @@ function toggle() { advanced.slideToggle(function(){ if (isHidden(this)) { + console.log('Setting storagekey to ', 0); localStorage.setItem(storageKey, '0'); } else { + console.log('Setting storagekey to ', 1); localStorage.setItem(storageKey, '1'); } }); @@ -38,11 +40,37 @@ toggle(); }); + console.log('storageKey', localStorage.getItem(storageKey)) // Show element if localstorage says so if (+localStorage.getItem(storageKey) === 1) { - toggle(); + advanced.show(); + fa.toggleClass('fa-caret-square-o-right fa-caret-square-o-down'); + } + + + // Master- and instancefield are mutually exclusive. Try to enforce that. + var masterField = document.querySelector('#id_master'); + var instanceField = document.querySelector('#id_virtual_instance'); + + function checkField(field1, field2){ + if (field1.value && !field1.disabled) { + field2.disabled = true; + } else { + field2.disabled = false; + } } + checkField(masterField, instanceField); + checkField(instanceField, masterField); + + $(masterField).change(function(){ + checkField(masterField, instanceField); + }); + + $(instanceField).change(function(){ + checkField(instanceField, masterField); + }); + {% endblock %} From 10ac2f7920b5702459c2ac2d34061699b2080bb5 Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 15:30:37 +0100 Subject: [PATCH 13/31] Fix label --- python/nav/web/seeddb/page/netbox/forms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index 8a18964d7e..6aa76e43da 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -55,6 +55,7 @@ class NetboxModelForm(forms.ModelForm): widget=forms.RadioSelect, initial='2') virtual_instance = MyModelMultipleChoiceField( queryset=Netbox.objects.none(), required=False, + label='Virtual instances', help_text='The list of virtual instances inside this master device') class Meta(object): From 2434344cda080605b9ce20ca969f8352a3546e6e Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Mon, 27 Mar 2017 10:32:03 +0200 Subject: [PATCH 14/31] Disable fields based on master/instance state --- htdocs/static/js/src/seeddb_netbox.js | 61 ++++++++++++++++++++ python/nav/web/seeddb/page/netbox/forms.py | 8 ++- templates/seeddb/netbox_wizard.html | 65 +--------------------- 3 files changed, 69 insertions(+), 65 deletions(-) create mode 100644 htdocs/static/js/src/seeddb_netbox.js diff --git a/htdocs/static/js/src/seeddb_netbox.js b/htdocs/static/js/src/seeddb_netbox.js new file mode 100644 index 0000000000..ee2f29bc42 --- /dev/null +++ b/htdocs/static/js/src/seeddb_netbox.js @@ -0,0 +1,61 @@ +require(['libs/select2.min'], function() { + $(function() { + var toggleTrigger = $('.advanced-toggle'), + fa = toggleTrigger.find('.fa'), + advanced = $('.advanced'), + storageKey = 'NAV.seeddb.advanced.show'; + + function isHidden(element) { + return element.offsetParent === null; + } + + function toggle() { + advanced.slideToggle(function(){ + if (isHidden(this)) { + console.log('Setting storagekey to ', 0); + localStorage.setItem(storageKey, '0'); + } else { + console.log('Setting storagekey to ', 1); + localStorage.setItem(storageKey, '1'); + } + }); + fa.toggleClass('fa-caret-square-o-right fa-caret-square-o-down'); + } + + toggleTrigger.on('click', function(event) { + event.preventDefault(); + toggle(); + }); + + // Show element if localstorage says so + if (+localStorage.getItem(storageKey) === 1) { + advanced.show(); + fa.toggleClass('fa-caret-square-o-right fa-caret-square-o-down'); + } + + + // Master- and instancefield are mutually exclusive. Try to enforce that. + var $masterField = $('#id_master').select2(); + var $instanceField = $('#id_virtual_instance').select2(); + + function checkFields() { + if ($masterField.val()) { + $instanceField.select2('enable', false); + } else { + $instanceField.select2('enable', true); + } + + if ($instanceField.val() && !$masterField[0].disabled) { + $masterField.select2('enable', false); + } else { + $masterField.select2('enable', true); + } + + + } + + $masterField.on('change', checkFields); + $instanceField.on('change', checkFields); + + }); +}); diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index 6aa76e43da..8373be06c0 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -78,12 +78,18 @@ def __init__(self, *args, **kwargs): Netbox.objects.filter(master__isnull=False)] self.fields['master'].queryset = self.create_master_query(masters) self.fields['virtual_instance'].queryset = self.create_instance_query(masters) + if self.instance.pk: # Set instances that we are master to as initial values self.initial['virtual_instance'] = Netbox.objects.filter( master=self.instance) - if self.instance.pk: + # Disable fields based on current state + if self.instance.master: + self.fields['virtual_instance'].widget.attrs['disabled'] = True + if self.instance.pk in masters: + self.fields['master'].widget.attrs['disabled'] = True + # Set the inital value of the function field try: netboxinfo = self.instance.info_set.get(variable='function') diff --git a/templates/seeddb/netbox_wizard.html b/templates/seeddb/netbox_wizard.html index f70f975e76..b63f8bbab7 100644 --- a/templates/seeddb/netbox_wizard.html +++ b/templates/seeddb/netbox_wizard.html @@ -7,70 +7,7 @@ NAV.urls.seeddb = NAV.urls.seeddb || {}; NAV.urls.get_readonly = "{% url 'seeddb-netbox-get-readonly' %}"; NAV.urls.seeddb.verifyAddress = "{% url 'seeddb-netbox-get-address-info' %}"; - - -{% endblock %} - -{% block footer_scripts %} - {% endblock %} From 9a89102af0cd7e2f4df09703f8493843efe1df1e Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Mon, 27 Mar 2017 10:49:51 +0200 Subject: [PATCH 15/31] Clean up leftover instances when saving --- python/nav/web/seeddb/page/netbox/forms.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index 8373be06c0..d8f0372f6a 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -225,6 +225,12 @@ def _check_existing_ip(self, ip): def save(self, commit=True): netbox = super(NetboxModelForm, self).save(commit) instances = self.cleaned_data.get('virtual_instance') + + # Clean up instances + Netbox.objects.filter( + master=netbox).exclude(pk__in=instances).update(master=None) + + # Add new instances for instance in instances: instance.master = netbox instance.save() From 95fcf8d3d08be0759a9144d2626e4cee59fdbcbc Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Tue, 21 Mar 2017 08:11:33 +0100 Subject: [PATCH 16/31] (Netbox.master): new field to enable a relation to physical masters i.e. in the case of VRF, the physical node is the master. Other modes of operation that are similar to VRF exist (including VDC). --- python/nav/models/manage.py | 4 +++- sql/changes/sc.04.07.0011.sql | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 sql/changes/sc.04.07.0011.sql diff --git a/python/nav/models/manage.py b/python/nav/models/manage.py index 1b7fd8516f..07d8713f1c 100644 --- a/python/nav/models/manage.py +++ b/python/nav/models/manage.py @@ -99,8 +99,10 @@ class Netbox(models.Model): up_since = models.DateTimeField(db_column='upsince', auto_now_add=True) up_to_date = models.BooleanField(db_column='uptodate', default=False) discovered = models.DateTimeField(auto_now_add=True) - deleted_at = models.DateTimeField(blank=True, null=True, default=None) + master = models.ForeignKey('Netbox', db_column='masterid', null=True, + blank=True, default=None, + related_name='instances') data = hstore.DictionaryField() objects = hstore.HStoreManager() diff --git a/sql/changes/sc.04.07.0011.sql b/sql/changes/sc.04.07.0011.sql new file mode 100644 index 0000000000..7cb687750c --- /dev/null +++ b/sql/changes/sc.04.07.0011.sql @@ -0,0 +1,10 @@ +ALTER TABLE netbox + ADD COLUMN masterid INTEGER DEFAULT NULL, + ADD CONSTRAINT netbox_masterid_fkey + FOREIGN KEY (masterid) + REFERENCES netbox(netboxid) + ON DELETE CASCADE ON UPDATE CASCADE; + +COMMENT ON COLUMN netbox.masterid IS + 'In the case of virtual sub-units, this field references the physical master unit'; + From fd31a0ff12d9dc95c9265a5034936cc257dc60a1 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Tue, 21 Mar 2017 14:04:14 +0100 Subject: [PATCH 17/31] (StatPorts): remove redundant collection in cases of virtualized instances In the case of a master netbox, duplicate all collected port metrics across all instance netboxes when shipping to Graphite. In the case of a virtualized instance, just debug log a list of local ports (that do not appear on the master) and exit. There is _no_ collection of data from local ports at the moment, as this would require a complete restructuring of things (i.e collection from individual ports, rather than full column snmpwalks for the values we want) --- python/nav/ipdevpoll/plugins/statports.py | 43 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/python/nav/ipdevpoll/plugins/statports.py b/python/nav/ipdevpoll/plugins/statports.py index ae0f093987..f8202f28cd 100644 --- a/python/nav/ipdevpoll/plugins/statports.py +++ b/python/nav/ipdevpoll/plugins/statports.py @@ -17,6 +17,7 @@ import time from twisted.internet import defer from nav.ipdevpoll import Plugin +from nav.ipdevpoll import db from nav.metrics.carbon import send_metrics from nav.metrics.templates import metric_path_for_interface from nav.mibs import reduce_index @@ -71,9 +72,15 @@ def can_handle(cls, netbox): @defer.inlineCallbacks def handle(self): + if self.netbox.master: + yield db.run_in_thread(self._log_instance_details) + defer.returnValue(None) + timestamp = time.time() stats = yield self._get_stats() - tuples = list(self._make_metrics(stats, timestamp)) + netboxes = yield db.run_in_thread(self._get_netbox_list) + tuples = list(self._make_metrics(stats, netboxes=netboxes, + timestamp=timestamp)) if tuples: self._logger.debug("Counters collected") send_metrics(tuples) @@ -95,7 +102,7 @@ def _get_stats(self): defer.returnValue(stats) - def _make_metrics(self, stats, timestamp=None): + def _make_metrics(self, stats, netboxes, timestamp=None): timestamp = timestamp or time.time() hc_counters = False @@ -104,11 +111,13 @@ def _make_metrics(self, stats, timestamp=None): for key in LOGGED_COUNTERS: if key not in row: continue - path = metric_path_for_interface( - self.netbox, row['ifName'] or row['ifDescr'], key) value = row[key] if value is not None: - yield (path, (timestamp, value)) + for netbox in netboxes: + # duplicate metrics for all involved netboxes + path = metric_path_for_interface( + netbox, row['ifName'] or row['ifDescr'], key) + yield (path, (timestamp, value)) if stats: if hc_counters: @@ -116,6 +125,30 @@ def _make_metrics(self, stats, timestamp=None): else: self._logger.debug("High Capacity counters NOT used") + def _get_netbox_list(self): + """Returns a list of netbox names to make metrics for. Will return just + the one netbox in most instances, but for situations with multiple + virtual device contexts, all the subdevices will be returned. + + """ + netboxes = [self.netbox.sysname] + instances = self.netbox.instances.values_list('sysname', flat=True) + netboxes.extend(instances) + self._logger.debug("duplicating metrics for these netboxes: %s", + netboxes) + return netboxes + + def _log_instance_details(self): + netbox = self.netbox + + my_ifcs = netbox.interface_set.values_list('ifname', flat=True) + masters_ifcs = netbox.master.interface_set.values_list('ifname', + flat=True) + local_ifcs = set(masters_ifcs) - set(my_ifcs) + + self._logger.debug("local interfaces (that do not exist on master " + "%s): %r", self.netbox.master, local_ifcs) + def use_hc_counters(row): """ From 490687c50f84d3ee472919829243604f3854025e Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Wed, 22 Mar 2017 13:45:54 +0100 Subject: [PATCH 18/31] (LinkState): modernize plugin to use inlineCallbacks --- python/nav/ipdevpoll/plugins/linkstate.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/nav/ipdevpoll/plugins/linkstate.py b/python/nav/ipdevpoll/plugins/linkstate.py index 7242b87e38..8e437d341a 100644 --- a/python/nav/ipdevpoll/plugins/linkstate.py +++ b/python/nav/ipdevpoll/plugins/linkstate.py @@ -14,24 +14,25 @@ # License along with NAV. If not, see . # """Collects interface link states and dispatches NAV events on changes""" +from twisted.internet.defer import inlineCallbacks, returnValue from nav.mibs import reduce_index from nav.mibs.if_mib import IfMib -from nav.models.manage import Interface - from nav.ipdevpoll import Plugin from nav.ipdevpoll import shadows class LinkState(Plugin): + """Monitors interface link states""" + @inlineCallbacks def handle(self): - self.ifmib = IfMib(self.agent) - df = self.ifmib.retrieve_columns( - ['ifName', 'ifAdminStatus', 'ifOperStatus']) - df.addCallback(reduce_index) - return df.addCallback(self._put_results) + ifmib = IfMib(self.agent) + result = yield ifmib.retrieve_columns( + ['ifName', 'ifAdminStatus', 'ifOperStatus']).addCallback( + reduce_index) + self._put_results(result) def _put_results(self, results): netbox = self.containers.factory(None, shadows.Netbox) From b6e67985a09b9791ea5cfad6d5164f6879c351bb Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 23 Mar 2017 10:45:58 +0100 Subject: [PATCH 19/31] do not poll interface status for virtualized instances. instead, have the eventengine make copies of linkState events for master devices. --- python/nav/eventengine/plugins/linkstate.py | 39 +++++++++++++++++++++ python/nav/ipdevpoll/plugins/linkstate.py | 5 +++ 2 files changed, 44 insertions(+) diff --git a/python/nav/eventengine/plugins/linkstate.py b/python/nav/eventengine/plugins/linkstate.py index 3201bd725a..3b4eb9f515 100644 --- a/python/nav/eventengine/plugins/linkstate.py +++ b/python/nav/eventengine/plugins/linkstate.py @@ -14,6 +14,7 @@ # along with NAV. If not, see . # """"linkState event plugin""" +import copy from nav.config import ConfigurationError from nav.eventengine.alerts import AlertGenerator @@ -44,6 +45,11 @@ def get_link_partner(self): """Returns the link partner of the target interface""" return self.get_target().to_netbox + def handle(self): + if self._is_a_master_for_virtualized_instances(): + self._copy_event_for_instances() + return super(LinkStateHandler, self).handle() + def _handle_end(self): self._post_event_if_aggregate_restored() # always verify aggregates return super(LinkStateHandler, self)._handle_end() @@ -160,6 +166,39 @@ def _get_aggregate_link_event(self, start): EventVar(event_queue=event, variable='aggregate_ifalias', value=target.ifalias or '').save() + # + # Methods to handle duplication of events for virtualized netbox instances + # + + def _is_a_master_for_virtualized_instances(self): + ifc = self.get_target() + return ifc and ifc.netbox and ifc.netbox.instances.count() > 0 + + def _copy_event_for_instances(self): + ifc = self.get_target() + netbox = ifc.netbox + for instance in netbox.instances.all(): + self._copy_event_for_instance(netbox, instance, ifc) + + def _copy_event_for_instance(self, netbox, instance, ifc): + try: + other_ifc = Interface.objects.get(netbox=instance, + ifname=ifc.ifname) + except Interface.DoesNotExist: + self._logger.info("interface %s does not exist on instance %s", + ifc.ifname, instance) + return + + new_event = copy.copy(self.event) # type: nav.models.event.EventQueue + new_event.pk = None + new_event.netbox = instance + new_event.device = None + new_event.subid = other_ifc.pk + + self._logger.info('duplicating linkState event for %s to %s', + ifc, instance) + new_event.save() + class LinkStateConfiguration(object): """Retrieves configuration options for the LinkStateHandler""" diff --git a/python/nav/ipdevpoll/plugins/linkstate.py b/python/nav/ipdevpoll/plugins/linkstate.py index 8e437d341a..7e7b21c2d9 100644 --- a/python/nav/ipdevpoll/plugins/linkstate.py +++ b/python/nav/ipdevpoll/plugins/linkstate.py @@ -28,6 +28,11 @@ class LinkState(Plugin): @inlineCallbacks def handle(self): + if self.netbox.master: + self._logger.debug("this is a virtual instance of %s, not polling", + self.netbox.master) + returnValue(None) + ifmib = IfMib(self.agent) result = yield ifmib.retrieve_columns( ['ifName', 'ifAdminStatus', 'ifOperStatus']).addCallback( From 3c3e89ced2cf1e761a63bb8cbac4142059ef71ed Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 09:49:16 +0100 Subject: [PATCH 20/31] (ipdevinfo): display related virtual instances/masters --- templates/ipdevinfo/frag-ipdevinfo.html | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/templates/ipdevinfo/frag-ipdevinfo.html b/templates/ipdevinfo/frag-ipdevinfo.html index fd1b642584..18b8be82d7 100644 --- a/templates/ipdevinfo/frag-ipdevinfo.html +++ b/templates/ipdevinfo/frag-ipdevinfo.html @@ -21,6 +21,13 @@ {{ netbox.sysname }} + {% if netbox.master %} + + Virtual instance of + {{ netbox.master }} + + {% endif %} + Type @@ -194,6 +201,23 @@ {{ netbox.get_gwports.count }} + {% if netbox.instances.count > 0 %} + + Virtual instances + + + + + {% endif %} + From eda89adfa6a0967121776f38022f9cce6d0dddfa Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 10:25:18 +0100 Subject: [PATCH 21/31] add snmp_version column to netbox bulk import format. --- NOTES.rst | 9 +++++++++ bin/dump.py | 3 ++- python/nav/bulkimport.py | 3 ++- python/nav/bulkparse.py | 12 +++++++++++- tests/integration/bulkimport_test.py | 15 ++++++++------- tests/unittests/general/bulkparse_test.py | 8 ++++---- 6 files changed, 36 insertions(+), 14 deletions(-) diff --git a/NOTES.rst b/NOTES.rst index eaea8b4d53..817485bbd2 100644 --- a/NOTES.rst +++ b/NOTES.rst @@ -52,6 +52,15 @@ are: .. _`Rittal liquid cooling package (in-row liquid coolers)`: http://www.rittal.com/com-en/product/list.action?categoryPath=/PG0001/PG0168KLIMA1/PGR1951KLIMA1/PG1023KLIMA1 +Changes to bulk import formats +------------------------------ + +The IP Device (Netbox) bulk import format has changed. A new column for snmp +version has been added, just before the SNMP read only community column. +Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the +bulk import format was not updated in the same release, so any device added +through the SeedDB bulk import function would default to SNMP v2c. + NAV 4.6 ======== diff --git a/bin/dump.py b/bin/dump.py index f36ef217d1..cf5fb10981 100755 --- a/bin/dump.py +++ b/bin/dump.py @@ -60,12 +60,13 @@ class Handlers(object): @staticmethod def netbox(): """Outputs a line for each netbox in the database""" - header("#roomid:ip:orgid:catid:[ro:rw:function:" + header("#roomid:ip:orgid:catid:[snmp_version:ro:rw:function:" "key1=value1|key2=value2:" "devicegroup1:devicegroup2..]") all_functions = manage.NetboxInfo.objects.filter(key='function') for box in manage.Netbox.objects.all(): line = [box.room_id, box.ip, box.organization_id, box.category_id, + str(box.snmp_version) if box.snmp_version else "", box.read_only or "", box.read_write or ""] functions = all_functions.filter(netbox=box) functions = str.join(", ", functions) diff --git a/python/nav/bulkimport.py b/python/nav/bulkimport.py index db98573c89..3efb7d0dd0 100644 --- a/python/nav/bulkimport.py +++ b/python/nav/bulkimport.py @@ -94,7 +94,8 @@ def _create_objects_from_row(self, row): @staticmethod def _get_netbox_from_row(row): netbox = Netbox(ip=row['ip'], read_only=row['ro'], - read_write=row['rw'], snmp_version=2) + read_write=row['rw'], + snmp_version=row['snmp_version'] or 2) netbox.room = get_object_or_fail(Room, id=row['roomid']) netbox.organization = get_object_or_fail(Organization, id=row['orgid']) netbox.category = get_object_or_fail(Category, id=row['catid']) diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index d7f82a0877..e05cabf5df 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -137,7 +137,8 @@ def next(self): class NetboxBulkParser(BulkParser): """Parses the netbox bulk format""" - format = ('roomid', 'ip', 'orgid', 'catid', 'ro', 'rw', 'function', 'data') + format = ('roomid', 'ip', 'orgid', 'catid', 'snmp_version', 'ro', 'rw', + 'function', 'data') required = 4 restkey = 'netboxgroup' @@ -150,6 +151,15 @@ def _validate_ip(value): else: return True + @staticmethod + def _validate_snmp_version(value): + if not value: + return True # empty values are ok + try: + return int(value) in (1,2) + except ValueError: + return False + @staticmethod def _validate_data(datastring): try: diff --git a/tests/integration/bulkimport_test.py b/tests/integration/bulkimport_test.py index cae6185316..c42dd1946b 100644 --- a/tests/integration/bulkimport_test.py +++ b/tests/integration/bulkimport_test.py @@ -21,7 +21,7 @@ def test_is_generator(self): class TestNetboxImporter(DjangoTransactionTestCase): def test_simple_import_yields_netbox_and_device_model(self): - data = 'myroom:10.0.90.252:myorg:SW:public::' + data = 'myroom:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -31,7 +31,7 @@ def test_simple_import_yields_netbox_and_device_model(self): self.assertTrue(isinstance(objects[0], manage.Netbox), objects[0]) def test_simple_import_yields_objects_with_proper_values(self): - data = 'myroom:10.0.90.252:myorg:SW:public::' + data = 'myroom:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -41,17 +41,18 @@ def test_simple_import_yields_objects_with_proper_values(self): self.assertEquals(netbox.room_id, 'myroom') self.assertEquals(netbox.organization_id, 'myorg') self.assertEquals(netbox.category_id, 'SW') + self.assertEquals(netbox.snmp_version, '1') self.assertEquals(netbox.read_only, 'public') def test_invalid_room_gives_error(self): - data = 'invalid:10.0.90.252:myorg:SW:public::' + data = 'invalid:10.0.90.252:myorg:SW:1:public::' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() self.assertTrue(isinstance(objects, DoesNotExist)) def test_netbox_function_is_set(self): - data = 'myroom:10.0.90.252:myorg:SW:public::does things:' + data = 'myroom:10.0.90.252:myorg:SW:1:public::does things:' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -69,7 +70,7 @@ def test_get_netboxinfo_from_function(self): self.assertEquals(netboxinfo.value, 'hella') def test_netbox_groups_are_set(self): - data = 'myroom:10.0.90.10:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -101,7 +102,7 @@ def test_duplicate_locations_should_give_error(self): snmp_version=1) netbox.save() - data = 'myroom:10.1.0.1:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.1.0.1:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -109,7 +110,7 @@ def test_duplicate_locations_should_give_error(self): self.assertTrue(isinstance(objects, AlreadyExists)) def test_created_objects_can_be_saved(self): - data = 'myroom:10.0.90.10:myorg:SRV:::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() diff --git a/tests/unittests/general/bulkparse_test.py b/tests/unittests/general/bulkparse_test.py index d72d353e17..8bf50e3c03 100644 --- a/tests/unittests/general/bulkparse_test.py +++ b/tests/unittests/general/bulkparse_test.py @@ -43,7 +43,7 @@ def test_parse_single_line_should_yield_value(self): self.assertTrue(out_data is not None) def test_parse_single_line_yields_columns(self): - data = ("room1:10.0.0.186:myorg:SW:public:secret:doesthings:" + data = ("room1:10.0.0.186:myorg:SW:1:public:secret:doesthings:" "key=value:blah1:blah2") b = NetboxBulkParser(data) out_data = b.next() @@ -59,10 +59,10 @@ def test_get_header(self): self.assertEquals( NetboxBulkParser.get_header(), "#roomid:ip:orgid:catid" - "[:ro:rw:function:data:netboxgroup:...]") + "[:snmp_version:ro:rw:function:data:netboxgroup:...]") def test_two_rows_returned_with_empty_lines_in_input(self): - data = ("room1:10.0.0.186:myorg:SW:public:parrot::\n" + data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" "\n" "room1:10.0.0.187:myorg:OTHER::parrot::\n") b = NetboxBulkParser(data) @@ -70,7 +70,7 @@ def test_two_rows_returned_with_empty_lines_in_input(self): self.assertEquals(len(out_data), 2) def test_three_lines_with_two_rows_should_be_counted_as_three(self): - data = ("room1:10.0.0.186:myorg:SW:public:parrot::\n" + data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" "\n" "room1:10.0.0.187:myorg:OTHER::parrot::\n") b = NetboxBulkParser(data) From 728325ffc07a7566eaf2332e871fc69a7b33510f Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 10:32:59 +0100 Subject: [PATCH 22/31] Add fields for choosing master and virtual instances on an IP Device --- python/nav/web/seeddb/page/netbox/forms.py | 73 +++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index 78f05b8094..fb5fa85dba 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -18,6 +18,7 @@ from socket import error as SocketError from django import forms +from django.db.models import Q from django_hstore.forms import DictionaryField from crispy_forms.helper import FormHelper from crispy_forms_foundation.layout import (Layout, Row, Column, Submit, @@ -33,6 +34,16 @@ _logger = logging.getLogger(__name__) +class MyModelMultipleChoiceField(forms.ModelMultipleChoiceField): + def __init__(self, queryset, cache_choices=False, required=True, + widget=None, label=None, initial=None, help_text='', *args, + **kwargs): + super(MyModelMultipleChoiceField, self).__init__( + queryset, cache_choices, required, widget, label, initial, + help_text, *args, **kwargs) + self.help_text = help_text + + class NetboxModelForm(forms.ModelForm): """Modelform for netbox for use in SeedDB""" ip = forms.CharField() @@ -42,18 +53,36 @@ class NetboxModelForm(forms.ModelForm): sysname = forms.CharField(required=False) snmp_version = forms.ChoiceField(choices=[('1', '1'), ('2', '2c')], widget=forms.RadioSelect, initial='2') + virtual_instance = MyModelMultipleChoiceField( + queryset=Netbox.objects.none(), required=False, + help_text='The list of virtual instances you are master to') class Meta(object): model = Netbox fields = ['ip', 'room', 'category', 'organization', 'read_only', 'read_write', 'snmp_version', - 'groups', 'sysname', 'type', 'data'] + 'groups', 'sysname', 'type', 'data', 'master', + 'virtual_instance'] + help_texts = { + 'master': 'Set the virtual master of this IP Device' + } def __init__(self, *args, **kwargs): super(NetboxModelForm, self).__init__(*args, **kwargs) self.fields['organization'].choices = create_hierarchy(Organization) + # Master and instance related queries + masters = [n.master.pk for n in + Netbox.objects.filter(master__isnull=False)] + self.fields['master'].queryset = self.create_master_query(masters) + self.fields['virtual_instance'].queryset = self.create_instance_query(masters) + if self.instance.pk: + # Set instances that we are master to as initial values + self.initial['virtual_instance'] = Netbox.objects.filter( + master=self.instance) + if self.instance.pk: + # Set the inital value of the function field try: netboxinfo = self.instance.info_set.get(variable='function') except NetboxInfo.DoesNotExist: @@ -95,12 +124,43 @@ def __init__(self, *args, **kwargs): Fieldset('Meta information', 'function', Field('groups', css_class='select2'), - 'data'), + 'data', + 'master', 'virtual_instance' + ), css_class=css_class), ), Submit('save_ip_device', 'Save IP device') ) + def create_instance_query(self, masters): + """Creates query for virtual instance multiselect""" + if self.instance.master: + # If we have a master, we should not be able to master instances + queryset = Netbox.objects.none() + else: + # - Should not see other masters + # - Should see those we are master for + # - Should see those who have no master + queryset = Netbox.objects.exclude(pk__in=masters).filter( + Q(master=self.instance.pk) | Q(master__isnull=True)) + + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) + + return queryset + + def create_master_query(self, masters): + """Creates query for master dropdown list""" + if self.instance and self.instance.pk in masters: + queryset = Netbox.objects.none() + else: + # - Should not set those who have master as master + queryset = Netbox.objects.filter(master__isnull=True) + if self.instance.pk: + queryset = queryset.exclude(pk=self.instance.pk) + + return queryset + def clean_ip(self): """Make sure IP-address is valid""" name = self.cleaned_data['ip'].strip() @@ -156,6 +216,15 @@ def _check_existing_ip(self, ip): if len(msg) > 0: raise IPExistsException(msg) + def save(self, commit=True): + netbox = super(NetboxModelForm, self).save(commit) + instances = self.cleaned_data.get('virtual_instance') + for instance in instances: + instance.master = netbox + instance.save() + + return netbox + class NetboxFilterForm(forms.Form): """Form for filtering netboxes on the list page""" From 9b8044d557f27a87d85bdddc1d606c79962c3820 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 10:54:59 +0100 Subject: [PATCH 23/31] pylint violation fix. --- python/nav/bulkparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index e05cabf5df..3b2e2e55f6 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -156,7 +156,7 @@ def _validate_snmp_version(value): if not value: return True # empty values are ok try: - return int(value) in (1,2) + return int(value) in (1, 2) except ValueError: return False From f905ccc2b62a017583a0b9f0ad7d3b812d482310 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 11:36:33 +0100 Subject: [PATCH 24/31] add a new master column to the netbox bulk import format. tests and doc updated. --- NOTES.rst | 22 +++++++++++++++++----- bin/dump.py | 3 ++- python/nav/bulkimport.py | 10 ++++++++++ python/nav/bulkparse.py | 2 +- tests/integration/bulkimport_test.py | 13 ++++++++++--- tests/unittests/general/bulkparse_test.py | 8 +++++--- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/NOTES.rst b/NOTES.rst index 817485bbd2..5c6c9180cf 100644 --- a/NOTES.rst +++ b/NOTES.rst @@ -55,12 +55,24 @@ are: Changes to bulk import formats ------------------------------ -The IP Device (Netbox) bulk import format has changed. A new column for snmp -version has been added, just before the SNMP read only community column. -Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the -bulk import format was not updated in the same release, so any device added -through the SeedDB bulk import function would default to SNMP v2c. +The IP Device (Netbox) bulk import format has changed. Two new columns have +been added, so that the format is now specified as:: + roomid:ip:orgid:catid[:snmp_version:ro:rw:master:function:data:netboxgroup:...] + +The new columns are: + +snmp_version + Selecting an explicit SNMP version was made compulsory in NAV 4.6, but the + bulk import format was not updated in the same release, so any device added + through the SeedDB bulk import function would default to SNMP v2c. Valid + values here are 1 or 2. + +master + If this device is a virtual instance on another physical device, specify the + sysname or IP address of the master in this column. You may have to bulk + import multiple times if the master devices are part of the same bulk import + file. NAV 4.6 ======== diff --git a/bin/dump.py b/bin/dump.py index cf5fb10981..d5dd4abd75 100755 --- a/bin/dump.py +++ b/bin/dump.py @@ -67,7 +67,8 @@ def netbox(): for box in manage.Netbox.objects.all(): line = [box.room_id, box.ip, box.organization_id, box.category_id, str(box.snmp_version) if box.snmp_version else "", - box.read_only or "", box.read_write or ""] + box.read_only or "", box.read_write or "", + box.master.sysname if box.master else ""] functions = all_functions.filter(netbox=box) functions = str.join(", ", functions) line.append(functions) diff --git a/python/nav/bulkimport.py b/python/nav/bulkimport.py index 3efb7d0dd0..772c7050d4 100644 --- a/python/nav/bulkimport.py +++ b/python/nav/bulkimport.py @@ -25,6 +25,7 @@ from nav.models.manage import Prefix, Vlan, NetType from nav.models.cabling import Cabling, Patch from nav.models.service import Service, ServiceProperty +from nav.util import is_valid_ip from nav.web.servicecheckers import get_description from nav.bulkparse import BulkParseError @@ -100,6 +101,15 @@ def _get_netbox_from_row(row): netbox.organization = get_object_or_fail(Organization, id=row['orgid']) netbox.category = get_object_or_fail(Category, id=row['catid']) netbox.sysname = netbox.ip + + master = row.get('master') + if master: + if is_valid_ip(master, use_socket_lib=True): + netbox.master = get_object_or_fail(Netbox, ip=master) + else: + netbox.master = get_object_or_fail(Netbox, + sysname__startswith=master) + return netbox @staticmethod diff --git a/python/nav/bulkparse.py b/python/nav/bulkparse.py index 3b2e2e55f6..2eb236bbc2 100644 --- a/python/nav/bulkparse.py +++ b/python/nav/bulkparse.py @@ -138,7 +138,7 @@ def next(self): class NetboxBulkParser(BulkParser): """Parses the netbox bulk format""" format = ('roomid', 'ip', 'orgid', 'catid', 'snmp_version', 'ro', 'rw', - 'function', 'data') + 'master', 'function', 'data') required = 4 restkey = 'netboxgroup' diff --git a/tests/integration/bulkimport_test.py b/tests/integration/bulkimport_test.py index c42dd1946b..7096176397 100644 --- a/tests/integration/bulkimport_test.py +++ b/tests/integration/bulkimport_test.py @@ -52,7 +52,7 @@ def test_invalid_room_gives_error(self): self.assertTrue(isinstance(objects, DoesNotExist)) def test_netbox_function_is_set(self): - data = 'myroom:10.0.90.252:myorg:SW:1:public::does things:' + data = 'myroom:10.0.90.252:myorg:SW:1:public:::does things:' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -70,7 +70,7 @@ def test_get_netboxinfo_from_function(self): self.assertEquals(netboxinfo.value, 'hella') def test_netbox_groups_are_set(self): - data = 'myroom:10.0.90.10:myorg:SRV::::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.0.90.10:myorg:SRV:::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -102,7 +102,7 @@ def test_duplicate_locations_should_give_error(self): snmp_version=1) netbox.save() - data = 'myroom:10.1.0.1:myorg:SRV::::fileserver::WEB:UNIX:MAIL' + data = 'myroom:10.1.0.1:myorg:SRV:::::fileserver::WEB:UNIX:MAIL' parser = NetboxBulkParser(data) importer = NetboxImporter(parser) _line_num, objects = importer.next() @@ -120,6 +120,13 @@ def test_created_objects_can_be_saved(self): print(repr(obj)) obj.save() + def test_invalid_master_should_give_error(self): + data = 'myroom:10.0.90.10:myorg:SW::::badmaster:functionality' + parser = NetboxBulkParser(data) + importer = NetboxImporter(parser) + _line_num, objects = importer.next() + self.assertTrue(isinstance(objects, DoesNotExist)) + class TestLocationImporter(DjangoTransactionTestCase): def test_import(self): diff --git a/tests/unittests/general/bulkparse_test.py b/tests/unittests/general/bulkparse_test.py index 8bf50e3c03..0afcb90428 100644 --- a/tests/unittests/general/bulkparse_test.py +++ b/tests/unittests/general/bulkparse_test.py @@ -23,13 +23,14 @@ def _validate_one(self, value): b = TestParser(data) try: list(b) - except InvalidFieldValue, error: + except InvalidFieldValue as error: self.assertEquals(error.line_num, 2) self.assertEquals(error.field, 'one') self.assertEquals(error.value, 'once') else: self.fail("No exception raised") + class TestNetboxBulkParser(TestCase): def test_parse_returns_iterator(self): data = "room1:10.0.0.186:myorg:OTHER::parrot::" @@ -43,7 +44,7 @@ def test_parse_single_line_should_yield_value(self): self.assertTrue(out_data is not None) def test_parse_single_line_yields_columns(self): - data = ("room1:10.0.0.186:myorg:SW:1:public:secret:doesthings:" + data = ("room1:10.0.0.186:myorg:SW:1:public:secret:amaster:doesthings:" "key=value:blah1:blah2") b = NetboxBulkParser(data) out_data = b.next() @@ -52,6 +53,7 @@ def test_parse_single_line_yields_columns(self): self.assertEquals(out_data['ip'], '10.0.0.186') self.assertEquals(out_data['orgid'], 'myorg') self.assertEquals(out_data['catid'], 'SW') + self.assertEquals(out_data['master'], 'amaster') self.assertEquals(out_data['data'], 'key=value') self.assertEquals(out_data['netboxgroup'], ['blah1', 'blah2']) @@ -59,7 +61,7 @@ def test_get_header(self): self.assertEquals( NetboxBulkParser.get_header(), "#roomid:ip:orgid:catid" - "[:snmp_version:ro:rw:function:data:netboxgroup:...]") + "[:snmp_version:ro:rw:master:function:data:netboxgroup:...]") def test_two_rows_returned_with_empty_lines_in_input(self): data = ("room1:10.0.0.186:myorg:SW:1:public:parrot::\n" From f20266ae876a8b08dd7d75f3bce97506f083ce18 Mon Sep 17 00:00:00 2001 From: John-Magne Bredal Date: Fri, 24 Mar 2017 13:36:26 +0100 Subject: [PATCH 25/31] Hide advanced options --- htdocs/sass/nav/seeddb.scss | 12 +++++--- python/nav/web/seeddb/page/netbox/forms.py | 10 ++++-- templates/seeddb/netbox_wizard.html | 36 ++++++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/htdocs/sass/nav/seeddb.scss b/htdocs/sass/nav/seeddb.scss index 032a368e30..4c7f9a7ba4 100644 --- a/htdocs/sass/nav/seeddb.scss +++ b/htdocs/sass/nav/seeddb.scss @@ -1,5 +1,9 @@ .readonly { - border: none; + border: none; +} + +.advanced { + display: none; } /* Style the tables on the index page */ @@ -21,9 +25,9 @@ } .required:after { - content: " *"; - color: #F00; - font-weight: bold; + content: " *"; + color: #F00; + font-weight: bold; } /* Style the general info tables */ diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index fb5fa85dba..ab3d70a239 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -22,7 +22,7 @@ from django_hstore.forms import DictionaryField from crispy_forms.helper import FormHelper from crispy_forms_foundation.layout import (Layout, Row, Column, Submit, - Fieldset, Field, Div) + Fieldset, Field, Div, HTML) from nav.web.crispyforms import LabelSubmit, NavButton from nav.models.manage import Room, Category, Organization, Netbox @@ -125,8 +125,12 @@ def __init__(self, *args, **kwargs): 'function', Field('groups', css_class='select2'), 'data', - 'master', 'virtual_instance' - ), + HTML(" Advanced options"), + Div( + 'master', 'virtual_instance', + css_class='advanced' + ) + ), css_class=css_class), ), Submit('save_ip_device', 'Save IP device') diff --git a/templates/seeddb/netbox_wizard.html b/templates/seeddb/netbox_wizard.html index 4bc7dee654..390f0daefc 100644 --- a/templates/seeddb/netbox_wizard.html +++ b/templates/seeddb/netbox_wizard.html @@ -7,6 +7,42 @@ NAV.urls.seeddb = NAV.urls.seeddb || {}; NAV.urls.get_readonly = "{% url 'seeddb-netbox-get-readonly' %}"; NAV.urls.seeddb.verifyAddress = "{% url 'seeddb-netbox-get-address-info' %}"; + + +{% endblock %} + +{% block footer_scripts %} + {% endblock %} From 129f59584febff8627a411807837916a6717cf72 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 24 Mar 2017 15:03:46 +0100 Subject: [PATCH 26/31] improve master/instance help texts slightly --- python/nav/web/seeddb/page/netbox/forms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/nav/web/seeddb/page/netbox/forms.py b/python/nav/web/seeddb/page/netbox/forms.py index ab3d70a239..b8515f2b92 100644 --- a/python/nav/web/seeddb/page/netbox/forms.py +++ b/python/nav/web/seeddb/page/netbox/forms.py @@ -55,7 +55,7 @@ class NetboxModelForm(forms.ModelForm): widget=forms.RadioSelect, initial='2') virtual_instance = MyModelMultipleChoiceField( queryset=Netbox.objects.none(), required=False, - help_text='The list of virtual instances you are master to') + help_text='The list of virtual instances inside this master device') class Meta(object): model = Netbox @@ -64,7 +64,8 @@ class Meta(object): 'groups', 'sysname', 'type', 'data', 'master', 'virtual_instance'] help_texts = { - 'master': 'Set the virtual master of this IP Device' + 'master': 'Select a master device when this IP Device is a virtual' + ' instance' } def __init__(self, *args, **kwargs): From 4f5bb731c813b3b6baa50e6fab5c7b3586f5717c Mon Sep 17 00:00:00 2001 From: Sigmund Augdal Date: Wed, 31 May 2017 11:54:04 +0200 Subject: [PATCH 27/31] (StatSensors): remove redundant collection in cases of virtualized instances --- python/nav/ipdevpoll/__init__.py | 13 +++++++++++++ python/nav/ipdevpoll/plugins/statports.py | 13 ------------- python/nav/ipdevpoll/plugins/statsensors.py | 15 ++++++++++----- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/python/nav/ipdevpoll/__init__.py b/python/nav/ipdevpoll/__init__.py index d22c536a6f..61a5fd75f3 100644 --- a/python/nav/ipdevpoll/__init__.py +++ b/python/nav/ipdevpoll/__init__.py @@ -97,3 +97,16 @@ def full_name(self): """Return the full module and class name of this instance.""" return "%s.%s" % (self.__class__.__module__, self.__class__.__name__) + + def _get_netbox_list(self): + """Returns a list of netbox names to make metrics for. Will return just + the one netbox in most instances, but for situations with multiple + virtual device contexts, all the subdevices will be returned. + + """ + netboxes = [self.netbox.sysname] + instances = self.netbox.instances.values_list('sysname', flat=True) + netboxes.extend(instances) + self._logger.debug("duplicating metrics for these netboxes: %s", + netboxes) + return netboxes diff --git a/python/nav/ipdevpoll/plugins/statports.py b/python/nav/ipdevpoll/plugins/statports.py index f8202f28cd..c05b2fd362 100644 --- a/python/nav/ipdevpoll/plugins/statports.py +++ b/python/nav/ipdevpoll/plugins/statports.py @@ -125,19 +125,6 @@ def _make_metrics(self, stats, netboxes, timestamp=None): else: self._logger.debug("High Capacity counters NOT used") - def _get_netbox_list(self): - """Returns a list of netbox names to make metrics for. Will return just - the one netbox in most instances, but for situations with multiple - virtual device contexts, all the subdevices will be returned. - - """ - netboxes = [self.netbox.sysname] - instances = self.netbox.instances.values_list('sysname', flat=True) - netboxes.extend(instances) - self._logger.debug("duplicating metrics for these netboxes: %s", - netboxes) - return netboxes - def _log_instance_details(self): netbox = self.netbox diff --git a/python/nav/ipdevpoll/plugins/statsensors.py b/python/nav/ipdevpoll/plugins/statsensors.py index 9736d0e865..26169a7d61 100644 --- a/python/nav/ipdevpoll/plugins/statsensors.py +++ b/python/nav/ipdevpoll/plugins/statsensors.py @@ -18,6 +18,7 @@ from twisted.internet import defer import time from nav.ipdevpoll import Plugin +from nav.ipdevpoll import db from nav.ipdevpoll.db import run_in_thread from nav.metrics.carbon import send_metrics from nav.metrics.templates import metric_path_for_sensor @@ -48,6 +49,9 @@ def _has_sensors(cls, netbox): @defer.inlineCallbacks def handle(self): + if self.netbox.master: + defer.returnValue(None) + netboxes = yield db.run_in_thread(self._get_netbox_list) sensors = yield run_in_thread(self._get_sensors) self._logger.debug("retrieving data from %d sensors", len(sensors)) oids = sensors.keys() @@ -55,23 +59,24 @@ def handle(self): for x in range(0, len(oids), MAX_SENSORS_PER_REQUEST)] for req in requests: data = yield self.agent.get(req).addCallback( - self._response_to_metrics, sensors) + self._response_to_metrics, sensors, netboxes) self._logger.debug("got data from sensors: %r", data) def _get_sensors(self): sensors = Sensor.objects.filter(netbox=self.netbox.id).values() return dict((row['oid'], row) for row in sensors) - def _response_to_metrics(self, result, sensors): + def _response_to_metrics(self, result, sensors, netboxes): metrics = [] timestamp = time.time() data = ((sensors[oid], value) for oid, value in result.iteritems() if oid in sensors) for sensor, value in data: value = convert_to_precision(value, sensor) - path = metric_path_for_sensor(self.netbox, - sensor['internal_name']) - metrics.append((path, (timestamp, value))) + for netbox in netboxes: + path = metric_path_for_sensor(netbox, + sensor['internal_name']) + metrics.append((path, (timestamp, value))) send_metrics(metrics) return metrics From e69cdc1a66c4b412ed246f97a6c2d416139d7285 Mon Sep 17 00:00:00 2001 From: Sigmund Augdal Date: Wed, 31 May 2017 13:09:40 +0200 Subject: [PATCH 28/31] (StatSystem): remove redundant collection in cases of virtualized instances --- python/nav/ipdevpoll/plugins/statsystem.py | 76 +++++++++++++--------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/python/nav/ipdevpoll/plugins/statsystem.py b/python/nav/ipdevpoll/plugins/statsystem.py index dad7208f00..3b72c1aaa2 100644 --- a/python/nav/ipdevpoll/plugins/statsystem.py +++ b/python/nav/ipdevpoll/plugins/statsystem.py @@ -20,6 +20,7 @@ from twisted.internet.error import TimeoutError from nav.ipdevpoll import Plugin +from nav.ipdevpoll import db from nav.metrics.carbon import send_metrics from nav.metrics.templates import ( metric_path_for_bandwith, @@ -65,20 +66,23 @@ class StatSystem(Plugin): """Collects system statistics and pushes to Graphite""" @defer.inlineCallbacks def handle(self): - bandwidth = yield self._collect_bandwidth() - cpu = yield self._collect_cpu() - sysuptime = yield self._collect_sysuptime() - memory = yield self._collect_memory() + if self.netbox.master: + defer.returnValue(None) + netboxes = yield db.run_in_thread(self._get_netbox_list) + bandwidth = yield self._collect_bandwidth(netboxes) + cpu = yield self._collect_cpu(netboxes) + sysuptime = yield self._collect_sysuptime(netboxes) + memory = yield self._collect_memory(netboxes) metrics = bandwidth + cpu + sysuptime + memory if metrics: send_metrics(metrics) @defer.inlineCallbacks - def _collect_bandwidth(self): + def _collect_bandwidth(self, netboxes): for mib in self._mibs_for_me(BANDWIDTH_MIBS): try: - metrics = yield self._collect_bandwidth_from_mib(mib) + metrics = yield self._collect_bandwidth_from_mib(mib, netboxes) except (TimeoutError, defer.TimeoutError): self._logger.debug("collect_bandwidth: ignoring timeout in %s", mib.mib['moduleName']) @@ -88,7 +92,7 @@ def _collect_bandwidth(self): defer.returnValue([]) @defer.inlineCallbacks - def _collect_bandwidth_from_mib(self, mib): + def _collect_bandwidth_from_mib(self, mib, netboxes): try: bandwidth = yield mib.get_bandwidth() bandwidth_peak = yield mib.get_bandwidth_peak() @@ -103,20 +107,22 @@ def _collect_bandwidth_from_mib(self, mib): mib.mib['moduleName'], bandwidth, bandwidth_peak) timestamp = time.time() - metrics = [ - (metric_path_for_bandwith(self.netbox, percent), - (timestamp, bandwidth)), - (metric_path_for_bandwith_peak(self.netbox, percent), - (timestamp, bandwidth_peak)), - ] + metrics = [] + for netbox in netboxes: + metrics += [ + (metric_path_for_bandwith(netbox, percent), + (timestamp, bandwidth)), + (metric_path_for_bandwith_peak(netbox, percent), + (timestamp, bandwidth_peak)), + ] defer.returnValue(metrics) @defer.inlineCallbacks - def _collect_cpu(self): + def _collect_cpu(self, netboxes): for mib in self._mibs_for_me(CPU_MIBS): try: - load = yield self._get_cpu_loadavg(mib) - utilization = yield self._get_cpu_utilization(mib) + load = yield self._get_cpu_loadavg(mib, netboxes) + utilization = yield self._get_cpu_utilization(mib, netboxes) except (TimeoutError, defer.TimeoutError): self._logger.debug("collect_cpu: ignoring timeout in %s", mib.mib['moduleName']) @@ -125,7 +131,7 @@ def _collect_cpu(self): defer.returnValue([]) @defer.inlineCallbacks - def _get_cpu_loadavg(self, mib): + def _get_cpu_loadavg(self, mib, netboxes): load = yield mib.get_cpu_loadavg() timestamp = time.time() metrics = [] @@ -135,13 +141,14 @@ def _get_cpu_loadavg(self, mib): mib.mib['moduleName'], load) for cpuname, loadlist in load.items(): for interval, value in loadlist: - path = metric_path_for_cpu_load(self.netbox, cpuname, - interval) - metrics.append((path, (timestamp, value))) + for netbox in netboxes: + path = metric_path_for_cpu_load(netbox, cpuname, + interval) + metrics.append((path, (timestamp, value))) defer.returnValue(metrics) @defer.inlineCallbacks - def _get_cpu_utilization(self, mib): + def _get_cpu_utilization(self, mib, netboxes): utilization = yield mib.get_cpu_utilization() timestamp = time.time() metrics = [] @@ -150,8 +157,9 @@ def _get_cpu_utilization(self, mib): self._logger.debug("Found CPU utilization from %s: %s", mib.mib['moduleName'], utilization) for cpuname, value in utilization.items(): - path = metric_path_for_cpu_utilization(self.netbox, cpuname) - metrics.append((path, (timestamp, value))) + for netbox in netboxes: + path = metric_path_for_cpu_utilization(netbox, cpuname) + metrics.append((path, (timestamp, value))) defer.returnValue(metrics) def _mibs_for_me(self, mib_class_dict): @@ -163,19 +171,22 @@ def _mibs_for_me(self, mib_class_dict): yield mib_class(self.agent) @defer.inlineCallbacks - def _collect_sysuptime(self): + def _collect_sysuptime(self, netboxes): mib = Snmpv2Mib(self.agent) uptime = yield mib.get_sysUpTime() timestamp = time.time() if uptime: - path = metric_path_for_sysuptime(self.netbox) - defer.returnValue([(path, (timestamp, uptime))]) + metrics = [] + for netbox in netboxes: + path = metric_path_for_sysuptime(netbox) + metrics.append((path, (timestamp, uptime))) + defer.returnValue(metrics) else: defer.returnValue([]) @defer.inlineCallbacks - def _collect_memory(self): + def _collect_memory(self, netboxes): memory = dict() for mib in self._mibs_for_me(MEMORY_MIBS): try: @@ -192,9 +203,10 @@ def _collect_memory(self): timestamp = time.time() result = [] for name, (used, free) in memory.items(): - prefix = metric_prefix_for_memory(self.netbox, name) - result.extend([ - (prefix + '.used', (timestamp, used)), - (prefix + '.free', (timestamp, free)), - ]) + for netbox in netboxes: + prefix = metric_prefix_for_memory(netbox, name) + result.extend([ + (prefix + '.used', (timestamp, used)), + (prefix + '.free', (timestamp, free)), + ]) defer.returnValue(result) From fd642c6b7f1deeeeab57808f4a0428bd5fe3ff29 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 2 Jun 2017 14:15:05 +0200 Subject: [PATCH 29/31] Enable access to the original model object in SeedDB list templates. SeedDB is, IMNSHO, overdue for a rewrite. The way it's written makes it really hard to do anything useful outside of building tables of text strings. This hack will at least enable any custom templates to access the original model object of each row, instead of a list of dicts containing compressed attribute strings :-P --- python/nav/web/seeddb/utils/list.py | 70 ++++++++++++++++------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/python/nav/web/seeddb/utils/list.py b/python/nav/web/seeddb/utils/list.py index a325cdde10..0f4dab5812 100644 --- a/python/nav/web/seeddb/utils/list.py +++ b/python/nav/web/seeddb/utils/list.py @@ -18,7 +18,8 @@ """Functions for rendering seeddb list views""" from collections import defaultdict -from django.core.urlresolvers import reverse +from django.core.urlresolvers import reverse, NoReverseMatch +from django.db.models import Model from django.shortcuts import render_to_response from django.template import RequestContext from django.db.models.fields import FieldDoesNotExist @@ -52,25 +53,18 @@ def render_list(request, queryset, value_list, edit_url=None, queryset = _filter_query(filter_form, queryset) - # Get values specified in value_list from the queryset. - # Also make sure that the primary key and the edit_url_attr appears. if not edit_url: - value_queryset = queryset.values('pk', *value_list) + rows, datakeys = _process_objects(queryset, value_list) else: - value_queryset = queryset.values('pk', edit_url_attr, *value_list) - - if not edit_url: - objects, datakeys = _process_objects(value_queryset, value_list) - else: - objects, datakeys = _process_objects(value_queryset, value_list, - edit_url, edit_url_attr) + rows, datakeys = _process_objects(queryset, value_list, + edit_url, edit_url_attr) labels = _label(queryset.model, value_list, datakeys) if add_descriptions: - _add_descriptions(objects, queryset) + _add_descriptions(rows, queryset) context = { - 'object_list': objects, + 'object_list': rows, 'labels': labels, 'filter_form': filter_form, 'sub_active': {'list': True}, @@ -95,8 +89,7 @@ def _filter_query(filter_form, queryset): return queryset -def _process_objects(query_set_values, value_list, edit_url=None, - edit_url_attr=None): +def _process_objects(queryset, value_list, edit_url=None, edit_url_attr=None): """Packs values into a format the template understands. A list contains each row. @@ -108,32 +101,49 @@ def _process_objects(query_set_values, value_list, edit_url=None, """ # pick up which values are dictionaries and make note of their existing keys datakeys = defaultdict(set) - for obj in query_set_values: + for obj in queryset: for attr in value_list: - value = obj[attr] + value = _getattr(obj, attr) if isinstance(value, dict): datakeys[attr].update(value) datakeys = dict([(k, list(sorted(v))) for k, v in datakeys.iteritems()]) def _getvalues(obj): for attr in value_list: - value = obj[attr] + value = _getattr(obj, attr) if attr in datakeys: for key in datakeys[attr]: yield value.get(key, None) else: yield value - objects = [] - for obj in query_set_values: + rows = [] + for obj in queryset: row = { - 'pk': obj['pk'], - 'values_list': list(_getvalues(obj)) + 'pk': obj.pk, + 'values_list': list(_getvalues(obj)), + 'model': obj, } - if edit_url: - row['url'] = reverse(edit_url, args=(obj[edit_url_attr],)) - objects.append(row) - return objects, datakeys + if edit_url and edit_url_attr: + key = _getattr(obj, edit_url_attr) + row['url'] = reverse(edit_url, args=(key,)) + rows.append(row) + return rows, datakeys + + +def _getattr(obj, attr): + """Deep getattr for Django double underscore specs. + + Should conform to the bassackwards ways of SeedDB at large. + """ + try: + value = reduce(getattr, attr.split('__'), obj) + if isinstance(value, Model): + return value.pk + else: + return value + except AttributeError: + pass def _label(model, value_list, datakeys=None): @@ -156,8 +166,8 @@ def _label(model, value_list, datakeys=None): return zip(labels, attrs) -def _add_descriptions(objects, queryset): +def _add_descriptions(rows, queryset): """Adds a description key to all objects""" - for obj in objects: - model = queryset.get(pk=obj['pk']) - obj['description'] = getattr(model, 'description', '') + for row in rows: + model = row['model'] + row['description'] = getattr(model, 'description', '') From 1bec10ab3fbabfd37b891949234885b1d0141d66 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 2 Jun 2017 14:18:37 +0200 Subject: [PATCH 30/31] Enable customization of just the row contents of SeedDB lists. --- templates/seeddb/list.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/seeddb/list.html b/templates/seeddb/list.html index 5ebd51c175..03f9913bb7 100644 --- a/templates/seeddb/list.html +++ b/templates/seeddb/list.html @@ -52,6 +52,7 @@ + {% block row %} {% for element in object.values_list %} {% if forloop.first %} @@ -69,6 +70,7 @@ {% endif %} {% endfor %} + {% endblock %} {% endfor %} {% else %} From 64dfccdac75b7f2cfe22457547cbafba11fe1b05 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Fri, 2 Jun 2017 14:19:11 +0200 Subject: [PATCH 31/31] Add VRF master/instance labels in the sysname column of SeedDB. Implemented as a custom template overriding the row logic for netbox lists. --- python/nav/web/seeddb/page/netbox/__init__.py | 1 + templates/seeddb/list_netbox.html | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 templates/seeddb/list_netbox.html diff --git a/python/nav/web/seeddb/page/netbox/__init__.py b/python/nav/web/seeddb/page/netbox/__init__.py index dc54b9b2b6..cc1a49ce4c 100644 --- a/python/nav/web/seeddb/page/netbox/__init__.py +++ b/python/nav/web/seeddb/page/netbox/__init__.py @@ -64,6 +64,7 @@ def netbox_list(request): return render_list(request, query, value_list, 'seeddb-netbox-edit', edit_url_attr='pk', filter_form=filter_form, + template='seeddb/list_netbox.html', extra_context=info.template_context, censor_list=create_index_list( value_list, diff --git a/templates/seeddb/list_netbox.html b/templates/seeddb/list_netbox.html new file mode 100644 index 0000000000..955789f73b --- /dev/null +++ b/templates/seeddb/list_netbox.html @@ -0,0 +1,31 @@ +{% extends "seeddb/list.html" %} +{% load url_parameters %} +{% load crispy_forms_tags %} + +{% block row %} + + {% for element in object.values_list %} + + {% if forloop.first %} {# first column is always sysname #} + + {{ element }} {{object.master }} + {% if object.model.master %} + V + {% elif object.model.instances.all|length > 0 %} + M + {% endif %} + + {% else %} + {% if element and forloop.counter0 in censor_list %} + ****** + {% else %} + {{ element|default_if_none:"" }} + {% endif %} + {% endif %} + + + {% endfor %} +{% endblock %}