Merge remote-tracking branch 'upstream/master'

rc9000 · Jan 14, 2019 · be6e178 · be6e178
2 parents bb61bca + c416e25
commit be6e178
Show file tree

Hide file tree

Showing 25 changed files with 214 additions and 124 deletions.
diff --git a/Changes b/Changes
@@ -1,19 +1,26 @@
 2.040003 - xxx
 
+  [NEW FEATURES]
+
+  * #482 operating system is now a link in device details (inphobia)
+
   [ENHANCEMENTS]
 
-  * #408 makerancidconf improvements readded with some additional tweaks, somehow it
-    got lost in 2.039029 (earendilfr, inphobia)
+  * #408, #417 & 477 makerancidconf improvements: readded with some additional tweaks, somehow
+    they got lost in 2.039029. (earendilfr, inphobia)
   * #436 make neighbor matching less strict, no longer is it required to match all parameters
     (port name, device name) exactly before creating a link between devices. (linwood-f)
 
   [BUG FIXES]
 
   * #475 all discover plugins should now respect ignore_interfaces, ignore_notpresent_types &
-    ignore_private_nets. as a bonus discovery data is now cached for less db load (ollyg)
+    ignore_private_nets. as a bonus discovery data is now cached for less db load. (ollyg)
   * #476 also log when plugins delete related tables. (ollyg)
-  * #471 redux ospf discovery will now keep on working instead of stopping after a few runs. (ollyg)
-  * better explain public key auth with netdisco-sshcollector (inphobia)
+  * #471-redux ospf discovery will now keep on working instead of stopping after a few
+    runs. (ollyg)
+  * #474 better explain public key auth with netdisco-sshcollector. (inphobia)
+  * #457 make sorting work on interface in device/adresses when interface was undefined. also
+    went through all other portsort callers to make sure parameters are always defined. (inphobia)
 
 2.040002 - 2018-12-30
 

diff --git a/lib/App/Netdisco/Web/GenericReport.pm b/lib/App/Netdisco/Web/GenericReport.pm
@@ -9,7 +9,7 @@ use App::Netdisco::Web::Plugin;
 use Path::Class 'file';
 use Safe;
 
-use vars qw/$config @data/;
+our ($config, @data);
 
 foreach my $report (@{setting('reports')}) {
   my $r = $report->{tag};

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm
@@ -114,6 +114,7 @@ sub store_neighbors {
     or return (); # already checked!
 
   # first allow any manually configured topology to be set
+  # and do this before we cache the rows in vars->{'device_ports'}
   set_manual_topology($device);
 
   if (!defined $snmp->has_topo) {
@@ -128,8 +129,9 @@ sub store_neighbors {
   my $c_platform = $snmp->c_platform;
 
   # cache the device ports to save hitting the database for many single rows
-  my $device_ports = vars->{'device_ports'}
-    || { map {($_->port => $_)} $device->ports->all };
+  vars->{'device_ports'} =
+    { map {($_->port => $_)} $device->ports->reset->all };
+  my $device_ports = vars->{'device_ports'};
 
   # v4 and v6 neighbor tables
   my $c_ip = ($snmp->c_ip || {});
@@ -153,7 +155,7 @@ sub store_neighbors {
       my $portrow = $device_ports->{$port};
 
       if (!defined $portrow) {
-          info sprintf ' [%s] neigh - local port %s not in database!',
+          debug sprintf ' [%s] neigh - local port %s already skipped, ignoring',
             $device->ip, $port;
           next;
       }
@@ -165,7 +167,7 @@ sub store_neighbors {
       }
 
       if ($portrow->manual_topo) {
-          info sprintf ' [%s] neigh - %s has manually defined topology',
+          debug sprintf ' [%s] neigh - %s has manually defined topology',
             $device->ip, $port;
           next;
       }
@@ -179,7 +181,7 @@ sub store_neighbors {
       my $r_netaddr = NetAddr::IP::Lite->new($remote_ip);
 
       if ($r_netaddr and ($r_netaddr->addr ne $remote_ip)) {
-        info sprintf ' [%s] neigh - IP on %s: using %s as canonical form of %s',
+        debug sprintf ' [%s] neigh - IP on %s: using %s as canonical form of %s',
           $device->ip, $port, $r_netaddr->addr, $remote_ip;
         $remote_ip = $r_netaddr->addr;
       }
@@ -193,7 +195,7 @@ sub store_neighbors {
           if ($remote_id) {
               my $devices = schema('netdisco')->resultset('Device');
               my $neigh = $devices->single({name => $remote_id});
-              info sprintf
+              debug sprintf
                 ' [%s] neigh - bad address %s on port %s, searching for %s instead',
                 $device->ip, $remote_ip, $port, $remote_id;
 
@@ -211,7 +213,7 @@ sub store_neighbors {
                   (my $tmpid = $remote_id) =~ s/.*\(([0-9a-f]{6})-([0-9a-f]{6})\).*/$1$2/;
                   my $mac = NetAddr::MAC->new(mac => $tmpid);
                   if ($mac and not $mac->errstr) {
-                      info sprintf
+                      debug sprintf
                         ' [%s] neigh - trying to find neighbor %s by MAC %s',
                         $device->ip, $remote_id, $mac->as_ieee;
                       $neigh = $devices->single({mac => $mac->as_ieee});
@@ -225,17 +227,17 @@ sub store_neighbors {
 
               if ($neigh) {
                   $remote_ip = $neigh->ip;
-                  info sprintf ' [%s] neigh - found %s with IP %s',
+                  debug sprintf ' [%s] neigh - found %s with IP %s',
                     $device->ip, $remote_id, $remote_ip;
               }
               else {
-                  info sprintf ' [%s] neigh - could not find %s, skipping',
+                  debug sprintf ' [%s] neigh - could not find %s, skipping',
                     $device->ip, $remote_id;
                   next;
               }
           }
           else {
-              info sprintf ' [%s] neigh - skipping unuseable address %s on port %s',
+              debug sprintf ' [%s] neigh - skipping unuseable address %s on port %s',
                 $device->ip, $remote_ip, $port;
               next;
           }
@@ -252,18 +254,18 @@ sub store_neighbors {
           $remote_port =~ s/[^\d\s\/\.,()\w:-]+//gi;
       }
       else {
-          info sprintf ' [%s] neigh - no remote port found for port %s at %s',
+          debug sprintf ' [%s] neigh - no remote port found for port %s at %s',
             $device->ip, $port, $remote_ip;
       }
 
-      $portrow->update({
+      $portrow = $portrow->update({
           remote_ip   => $remote_ip,
           remote_port => $remote_port,
           remote_type => $remote_type,
           remote_id   => $remote_id,
           is_uplink   => \"true",
           manual_topo => \"false",
-      });
+      })->discard_changes();
 
       # update master of our aggregate to be a neighbor of
       # the master on our peer device (a lot of iffs to get there...).

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm b/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm
@@ -50,7 +50,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
       my $port = $interfaces->{ $p_ifindex->{$entry} } or next;
 
       if (!defined $device_ports->{$port}) {
-          info sprintf ' [%s] power - local port %s not in database!',
+          debug sprintf ' [%s] power - local port %s already skipped, ignoring',
             $device->ip, $port;
           next;
       }

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm
@@ -30,7 +30,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   foreach my $idx (keys %$raw_speed) {
     my $port = $interfaces->{$idx} or next;
     if (!defined $device_ports->{$port}) {
-        info sprintf ' [%s] properties/speed - local port %s not in database!',
+        debug sprintf ' [%s] properties/speed - local port %s already skipped, ignoring',
           $device->ip, $port;
         next;
     }
@@ -43,7 +43,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   foreach my $idx (keys %$err_cause) {
     my $port = $interfaces->{$idx} or next;
     if (!defined $device_ports->{$port}) {
-        info sprintf ' [%s] properties/errdis - local port %s not in database!',
+        debug sprintf ' [%s] properties/errdis - local port %s already skipped, ignoring',
           $device->ip, $port;
         next;
     }
@@ -56,7 +56,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   foreach my $idx (keys %$faststart) {
     my $port = $interfaces->{$idx} or next;
     if (!defined $device_ports->{$port}) {
-        info sprintf ' [%s] properties/faststart - local port %s not in database!',
+        debug sprintf ' [%s] properties/faststart - local port %s already skipped, ignoring',
           $device->ip, $port;
         next;
     }
@@ -77,7 +77,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   foreach my $idx (keys %$c_if) {
     my $port = $interfaces->{ $c_if->{$idx} } or next;
     if (!defined $device_ports->{$port}) {
-        info sprintf ' [%s] properties/lldpcap - local port %s not in database!',
+        debug sprintf ' [%s] properties/lldpcap - local port %s already skipped, ignoring',
           $device->ip, $port;
         next;
     }
@@ -106,12 +106,12 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
 
   schema('netdisco')->txn_do(sub {
     my $gone = $device->properties_ports->delete;
-    debug sprintf ' [%s] props - removed %d ports with properties',
+    debug sprintf ' [%s] properties - removed %d ports with properties',
       $device->ip, $gone;
     $device->properties_ports->populate(
       [map {{ port => $_, %{ $properties{$_} } }} keys %properties] );
 
-    return Status->info(sprintf ' [%s] props - added %d new port properties',
+    return Status->info(sprintf ' [%s] properties - added %d new port properties',
       $device->ip, scalar keys %properties);
   });
 });

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm
@@ -151,7 +151,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
   if (defined $snmp->snmpEngineTime) {
       $dev_uptime_wrapped = int( $snmp->snmpEngineTime * 100 / 2**32 );
       if ($dev_uptime_wrapped > 0) {
-          info sprintf ' [%s] interface - device uptime wrapped %d times - correcting',
+          debug sprintf ' [%s] interfaces - device uptime wrapped %d times - correcting',
             $device->ip, $dev_uptime_wrapped;
           $device->uptime( $dev_uptime + $dev_uptime_wrapped * 2**32 );
       }
@@ -190,7 +190,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
 
       my $lc = $i_lastchange->{$entry} || 0;
       if (not $dev_uptime_wrapped and $lc > $dev_uptime) {
-          info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting',
+          debug sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting',
             $device->ip, $port;
           $device->uptime( $dev_uptime + 2**32 );
           $dev_uptime_wrapped = 1;
@@ -252,10 +252,6 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
     $device->update_or_insert(undef, {for => 'update'});
     $device->ports->populate([values %interfaces]);
 
-    # cache for later phases of the discovery
-    vars->{'device_ports'} =
-      { map {($_->port => $_)} $device->ports->all };
-
     return Status->info(sprintf ' [%s] interfaces - added %d new interfaces',
       $device->ip, scalar values %interfaces);
   });

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm b/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm
@@ -49,7 +49,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
       my $port = $interfaces->{$entry} or next;
 
       if (!defined $device_ports->{$port}) {
-          info sprintf ' [%s] vlans - local port %s not in database!',
+          debug sprintf ' [%s] vlans - local port %s already skipped, ignoring',
             $device->ip, $port;
           next;
       }

diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm
@@ -41,7 +41,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
       }
 
       if (!defined $device_ports->{$port}) {
-          info sprintf ' [%s] wireless - local port %s not in database!',
+          debug sprintf ' [%s] wireless - local port %s already skipped, ignoring',
             $device->ip, $port;
           next;
       }
@@ -75,7 +75,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
       }
 
       if (!defined $device_ports->{$port}) {
-          info sprintf ' [%s] wireless - local port %s not in database!',
+          debug sprintf ' [%s] wireless - local port %s already skipped, ignoring',
             $device->ip, $port;
           next;
       }

diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm
@@ -248,7 +248,6 @@ sub get_vlan_list {
 
       # check in use by a port on this device
       if (!$vlans{$vlan} && !setting('macsuck_all_vlans')) {
-
           debug sprintf
             ' [%s] macsuck VLAN %s/%s - not in use by any port - skipping.',
             $device->ip, $vlan, $name;