macsuck_unsupported setting to allow node gathering on delinquent switches

Netdisco/Changes

@@ -1,8 +1,12 @@
-2.031013 - 2015-03-02
+2.031013_001 - 2015-03-02
+
+  [NEW FEATURES]
+
+  * macsuck_unsupported setting to allow node gathering on delinquent switches
 
   [BUG FIXES]
 
-  * Only exclude discover_no on Undiscovered Neighbors report when few results
+  * Only exclude discover_no on Undiscovered Neighbors report when few (<50) results
 
 2.031012 - 2015-02-28
 

Netdisco/lib/App/Netdisco/Core/Macsuck.pm

@@ -4,6 +4,7 @@ use Dancer qw/:syntax :script/;
 use Dancer::Plugin::DBIC 'schema';
 
 use App::Netdisco::Util::PortMAC 'get_port_macs';
+use App::Netdisco::Util::Device qw/check_device_no match_devicetype/;
 use App::Netdisco::Util::Node 'check_mac';
 use App::Netdisco::Util::SNMP 'snmp_comm_reindex';
 use Time::HiRes 'gettimeofday';
@@ -67,7 +68,7 @@ sub do_macsuck {
 
   # cache the device ports to save hitting the database for many single rows
   my $device_ports = {map {($_->port => $_)}
-                          $device->ports(undef, {prefetch => 'neighbor_alias'})->all};
+                          $device->ports(undef, {prefetch => {neighbor_alias => 'device'}})->all};
   my $port_macs = get_port_macs();
   my $interfaces = $snmp->interfaces;
 
@@ -151,7 +152,8 @@ sub store_node {
 
     my $old = $nodes->search(
         { mac   => $mac,
-          vlan   => $vlan,
+          # where vlan is unknown, need to archive on all other vlans
+          ($vlan ? (vlan => $vlan) : ()),
           -bool => 'active',
           -not  => {
                     switch => $ip,
@@ -361,8 +363,21 @@ sub _walk_fwtable {
       #  * a mac addr is seen which belongs to any device port/interface
       #  * (TODO) admin sets is_uplink_admin on the device_port
 
+      # allow to gather MACs on upstream port for some kinds of device that
+      # do not expose MAC address tables via SNMP. relies on prefetched
+      # neighbors otherwise it would kill the DB with device lookups.
+      my $neigh_cannot_macsuck = eval { # can fail
+          check_device_no($device_port->neighbor, 'macsuck_unsupported') ||
+          match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') };
+
       if ($device_port->is_uplink) {
-          if (my $neighbor = $device_port->neighbor) {
+          if ($neigh_cannot_macsuck) {
+              debug sprintf
+                ' [%s] macsuck %s - port %s neighbor %s without macsuck support',
+                $device->ip, $mac, $port, $device_port->neighbor->ip;
+              # continue!!
+          }
+          elsif (my $neighbor = $device_port->neighbor) {
               debug sprintf
                 ' [%s] macsuck %s - port %s has neighbor %s - skipping.',
                 $device->ip, $mac, $port, $neighbor->ip;
@@ -398,6 +413,11 @@ sub _walk_fwtable {
             $device->ip, $mac, $port;
           $device_port->update({is_uplink => \'true'});
 
+          # neighbor exists and Netdisco can speak to it, so we don't want
+          # its MAC address. however don't add to skiplist as that would
+          # clear all other MACs on the port.
+          next if $neigh_cannot_macsuck;
+
           # when there's no CDP/LLDP, we only want to gather macs at the
           # topology edge, hence skip ports with known device macs.
           if (not setting('macsuck_bleed')) {

Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm

@@ -237,8 +237,16 @@ database.
 
 =cut
 
-__PACKAGE__->has_many( neighbor_alias => 'App::Netdisco::DB::Result::DeviceIp',
+__PACKAGE__->belongs_to( neighbor_alias => 'App::Netdisco::DB::Result::DeviceIp',
-  { 'foreign.alias' => 'self.remote_ip' },
+  sub {
+      my $args = shift;
+      return {
+          "$args->{foreign_alias}.ip" => { '=' =>
+            $args->{self_resultsource}->schema->resultset('DeviceIp')
+              ->search({alias => { -ident => "$args->{self_alias}.remote_ip"}},
+                       {rows => 1, columns => 'ip', alias => 'devipsub'})->as_query }
+      };
+  },
   { join_type => 'LEFT' },
 );
 
@@ -289,7 +297,7 @@ the database.
 
 sub neighbor {
     my $row = shift;
-    return eval { $row->neighbor_alias->first->device || undef };
+    return eval { $row->neighbor_alias->device || undef };
 }
 
 =head1 ADDITIONAL COLUMNS

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -737,6 +737,27 @@ Value: List of "IP:vlan-number" or "IP:vlan-name". Default: Empty List.
 Similar to C<macsuck_no_vlan>, but allows specifying the device root
 (canonical) IP, in order to restrict VLAN skipping only to some devices.
 
+=head3 C<macsuck_unsupported>
+
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
+
+Similar to C<macsuck_no>, but instead of skipping nodes on this device, they
+are allowed to gather on the upstream device port. Useful for devices which
+can be discovered by Netdisco but do not provide a MAC address table via SNMP.
+
+=head3 C<macsuck_unsupported_type>
+
+Value: List of Strings. Default: None.
+
+Place regular expression patterns here to skip macsuck of certain devices
+based on the CDP/LLDP device type information they advertise. MAC addresses
+will be allowed to gather on the upstream device port, as in the
+C<macscuk_unsupported> setting. For example:
+
+ macsuck_unsupported_type:
+   - 'cisco\s+AIR-LAP'
+   - '(?i)Cisco\s+IP\s+Phone'
+
 =head3 C<macsuck_bleed>
 
 Value: Boolean. Default: C<false>.

Netdisco/lib/App/Netdisco/Util/Device.pm

@@ -10,6 +10,7 @@ our @EXPORT_OK = qw/
   get_device
   delete_device
   renumber_device
+  match_devicetype
   check_device_no
   check_device_only
   is_discoverable
@@ -126,6 +127,21 @@ sub renumber_device {
   return $happy;
 }
 
+=head2 match_devicetype( $type, $setting_name )
+
+Given a C<$type> (which may be any text value), returns true if any of the
+list of regular expressions in C<$setting_name> is matched, otherwise returns
+false.
+
+=cut
+
+sub match_devicetype {
+    my ($type, $setting_name) = @_;
+    return 0 unless $type and $setting_name;
+    return (scalar grep {$type =~ m/$_/}
+                        @{setting($setting_name) || []});
+}
+
 =head2 check_device_no( $ip, $setting_name )
 
 Given the IP address of a device, returns true if the configuration setting
@@ -170,6 +186,8 @@ To match no devices we recommend an entry of "C<localhost>" in the setting.
 
 sub check_device_no {
   my ($ip, $setting_name) = @_;
+
+  return 0 unless $ip and $setting_name;
   my $device = get_device($ip) or return 0;
 
   my $config = setting($setting_name) || [];
@@ -251,10 +269,8 @@ sub is_discoverable {
   my ($ip, $remote_type) = @_;
   my $device = get_device($ip) or return 0;
 
-  if ($remote_type) {
+  if (match_devicetype($remote_type, 'discover_no_type')) {
-      return _bail_msg("is_discoverable: device matched discover_no_type")
+      return _bail_msg("is_discoverable: device matched discover_no_type");
-        if scalar grep {$remote_type =~ m/$_/}
-                    @{setting('discover_no_type') || []};
   }
 
   return _bail_msg("is_discoverable: device matched discover_no")

Netdisco/share/config.yml

@@ -136,6 +136,8 @@ macsuck_no_vlan:
   - 'fddinet-default'
   - 'trnet-default'
 macsuck_no_devicevlan: []
+macsuck_unsupported: []
+macsuck_unsupported_type: []
 macsuck_bleed: false
 macsuck_min_age: 0
 snmpforce_v1: []