Support for nonincreasing, bulkwalk_* settings and also property:match values

Netdisco/Changes

@@ -7,6 +7,7 @@
   * Support for discover_min_age, macsuck_min_age, arpnip_min_age
   * Support for macsuck_no, macsuck_only, arpnip_no, arpnip_only
   * Support for macsuck_no_vlan and macsuck_no_devicevlan
+  * Support for nonincreasing, bulkwalk_* settings and also property:match values
 
   [BUG FIXES]
 

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -258,6 +258,40 @@ Value: List of Strings. Default: C<private>.
 A list of read-write SNMP community strings to try on each device. The working
 community will be cached in the database.
 
+=head3 C<bulkwalk_off>
+
+Value: Boolean. Default C<false>.
+
+Set to C<true> to use C<GETNEXT> instead of the standard C<BULKWALK> for every
+device. This will slow things down, but might be necessary for problem
+devices. For more fine-grained control see the C<bulkwalk_no> setting.
+
+=head3 C<bulkwalk_no>
+
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
+
+IP addresses in the list will use C<GETNEXT> (and not C<BULKWALK>). You can
+include hostnames, IP addresses and subnets (IPv4 or IPv6) in the list.
+
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
+=head3 C<bulkwalk_repeaters>
+
+Value: Numnber. Default: 20.
+
+Sets the Net-SNMP C<MaxRepeaters> value, which is used on C<BULKWALK>
+operations. See L<SNMP> for more info.
+
+=head3 C<nonincreasing>
+
+Value: Boolean. Default: C<false>.
+
+Setting this to C<true> will allow the bulkwalk of devices that have tables
+with non-increasing OIDs. The default is to not allow this behavior, to
+prevent discovery on problem devices from looping indefinitely. Requires
+Net-SNMP 5.3 or higher.
+
 =head3 C<snmpver>
 
 Value: C<1|2|3>. Default: 2.
@@ -279,19 +313,25 @@ Number of times to retry connecting to a device before giving up.
 
 =head3 C<discover_no>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 IP addresses in the list will not be visited during device discovery. You can
 include hostnames, IP addresses and subnets (IPv4 or IPv6) in the list.
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<discover_only>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 If present, device discovery will be limited to IP addresses matching entries
 in this list. You can include hostnames, IP addresses and subnets (IPv4 and
 IPv6).
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<discover_no_type>
 
 Value: List of Strings. Default: None.
@@ -314,18 +354,24 @@ discover jobs for a device.
 
 =head3 C<macsuck_no>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 IP addresses in the list will not be visited for macsuck. You can include
 hostnames, IP addresses and subnets (IPv4 or IPv6) in the list.
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<macsuck_only>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 If present, macsuck will be limited to IP addresses matching entries in this
 list. You can include hostnames, IP addresses and subnets (IPv4 and IPv6).
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<macsuck_all_vlans>
 
 Value: Boolean. Default: C<false>.
@@ -373,18 +419,24 @@ macsuck jobs for a device.
 
 =head3 C<arpnip_no>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 IP addresses in the list will not be visited for arpnip. You can include
 hostnames, IP addresses and subnets (IPv4 or IPv6) in the list.
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<arpnip_only>
 
-Value: List of Network Identifiers. Default: Empty List.
+Value: List of Network Identifiers or Device Properties. Default: Empty List.
 
 If present, arpnip will be limited to IP addresses matching entries in this
 list. You can include hostnames, IP addresses and subnets (IPv4 and IPv6).
 
+Alternatively include a "C<property:regex>" entry to match the named property
+of the device. The regex must match the complete value.
+
 =head3 C<arpnip_min_age>
 
 Value: Number. Default: 0.
@@ -625,18 +677,6 @@ These settings are from Netdisco 1.x but are yet to be supported in Netdisco
 
 =item *
 
-C<bulkwalk_no>
-
-=item *
-
-C<bulkwalk_off>
-
-=item *
-
-C<bulkwalk_repeaters>
-
-=item *
-
 C<col_xxx_show>
 
 =item *
@@ -661,10 +701,6 @@ C<macsuck_timeout>
 
 =item *
 
-C<nonincreasing>
-
-=item *
-
 C<port_info>
 
 =item *

Netdisco/lib/App/Netdisco/Util/Device.pm

@@ -9,6 +9,7 @@ use base 'Exporter';
 our @EXPORT = ();
 our @EXPORT_OK = qw/
   get_device
+  check_no
   is_discoverable
   is_arpnipable
   is_macsuckable
@@ -56,6 +57,65 @@ sub get_device {
     ->find_or_new({ip => $ip});
 }
 
+=head2 check_no( $ip, $setting_name )
+
+Given the IP address of a device, returns true if the configuration setting
+C<$setting_name> matches that device, else returns false.
+
+There are several options for what C<$setting_name> can contain:
+
+=over 4
+
+=item *
+
+Hostname, IP address, IP prefix
+
+=item *
+
+C<"model:regex"> - matched against the device model
+
+=item *
+
+C<"vendor:regex"> - matched against the device vendor
+
+=back
+
+To simply match all devices, use IP Prefix "C<0.0.0.0/0>". All regular
+expressions are anchored (that is, they must match the whole string).
+
+=cut
+
+sub check_no {
+  my ($ip, $setting_name) = @_;
+  my $device = get_device($ip) or return 0;
+  my $addr = NetAddr::IP::Lite->new($device->ip);
+
+  my $config = setting($setting_name) || [];
+  return 0 unless scalar @$config;
+
+  foreach my $item (@$config) {
+      if ($item =~ m/^(.*)\s*:\s*(.*)$/) {
+          my $prop  = $1;
+          my $match = $2;
+
+          # if not in storage, we can't do much with device properties
+          next unless $device->in_storage;
+
+          # lazy version of vendor: and model:
+          if ($device->can($prop) and defined $device->prop
+              and $device->prop =~ m/^$match$/) {
+              return 1;
+          }
+          next;
+      }
+
+      my $ip = NetAddr::IP::Lite->new($item) or next;
+      return 1 if $ip->contains($addr);
+  }
+
+  return 0;
+}
+
 =head2 is_discoverable( $ip, $device_type? )
 
 Given an IP address, returns C<true> if Netdisco on this host is permitted by
@@ -83,32 +143,14 @@ sub is_discoverable {
                     @{setting('discover_no_type') || []};
   }
 
-  my $addr = NetAddr::IP::Lite->new($device->ip);
+  return _bail_msg("is_discoverable: device matched discover_no")
-  my $discover_no   = setting('discover_no') || [];
+    if check_no($device, 'discover_no');
-  my $discover_only = setting('discover_only') || [];
 
-  if (scalar @$discover_no) {
+  return _bail_msg("is_discoverable: device failed to match discover_only")
-      foreach my $item (@$discover_no) {
+    if check_no($device, 'discover_only');
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          return _bail_msg("is_discoverable: device matched discover_no")
-            if $ip->contains($addr);
-      }
-  }
 
-  if (scalar @$discover_only) {
+  if ($device->since_last_discover and setting('discover_min_age')
-      my $okay = 0;
+      and $device->since_last_discover < setting('discover_min_age')) {
-      foreach my $item (@$discover_only) {
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          ++$okay if $ip->contains($addr);
-      }
-      return _bail_msg("is_discoverable: device failed to match discover_only")
-        if not $okay;
-  }
-
-  my $discover_since = setting('discover_min_age') || 0;
-
-  if ($device->since_last_discover
-      and $device->since_last_discover < $discover_since) {
 
       return _bail_msg("is_discoverable: time since last discover less than discover_min_age");
   }
@@ -132,30 +174,14 @@ sub is_arpnipable {
   my $ip = shift;
   my $device = get_device($ip) or return 0;
 
-  my $addr = NetAddr::IP::Lite->new($device->ip);
+  return _bail_msg("is_arpnipable: device matched arpnip_no")
-  my $arpnip_no   = setting('arpnip_no') || [];
+    if check_no($device, 'arpnip_no');
-  my $arpnip_only = setting('arpnip_only') || [];
 
-  if (scalar @$arpnip_no) {
+  return _bail_msg("is_arpnipable: device failed to match arpnip_only")
-      foreach my $item (@$arpnip_no) {
+    if check_no($device, 'arpnip_only');
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          return 0 if $ip->contains($addr);
-      }
-  }
 
-  if (scalar @$arpnip_only) {
+  if ($device->since_last_arpnip and setting('arpnip_min_age')
-      my $okay = 0;
+      and $device->since_last_arpnip < setting('arpnip_min_age')) {
-      foreach my $item (@$arpnip_only) {
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          ++$okay if $ip->contains($addr);
-      }
-      return 0 if not $okay;
-  }
-
-  my $arpnip_since = setting('arpnip_min_age') || 0;
-
-  if ($device->since_last_arpnip
-      and $device->since_last_arpnip < $arpnip_since) {
 
       return _bail_msg("is_arpnipable: time since last arpnip less than arpnip_min_age");
   }
@@ -179,30 +205,14 @@ sub is_macsuckable {
   my $ip = shift;
   my $device = get_device($ip) or return 0;
 
-  my $addr = NetAddr::IP::Lite->new($device->ip);
+  return _bail_msg("is_macsuckable: device matched macsuck_no")
-  my $macsuck_no   = setting('macsuck_no') || [];
+    if check_no($device, 'macsuck_no');
-  my $macsuck_only = setting('macsuck_only') || [];
 
-  if (scalar @$macsuck_no) {
+  return _bail_msg("is_macsuckable: device failed to match macsuck_only")
-      foreach my $item (@$macsuck_no) {
+    if check_no($device, 'macsuck_only');
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          return 0 if $ip->contains($addr);
-      }
-  }
 
-  if (scalar @$macsuck_only) {
+  if ($device->since_last_macsuck and setting('macsuck_min_age')
-      my $okay = 0;
+      and $device->since_last_macsuck < setting('macsuck_min_age')) {
-      foreach my $item (@$macsuck_only) {
-          my $ip = NetAddr::IP::Lite->new($item) or return 0;
-          ++$okay if $ip->contains($addr);
-      }
-      return 0 if not $okay;
-  }
-
-  my $macsuck_since = setting('macsuck_min_age') || 0;
-
-  if ($device->since_last_macsuck
-      and $device->since_last_macsuck < $macsuck_since) {
 
       return _bail_msg("is_macsuckable: time since last macsuck less than macsuck_min_age");
   }

Netdisco/lib/App/Netdisco/Util/SNMP.pm

@@ -1,7 +1,7 @@
 package App::Netdisco::Util::SNMP;
 
 use Dancer qw/:syntax :script/;
-use App::Netdisco::Util::Device 'get_device';
+use App::Netdisco::Util::Device qw/get_device check_no/;
 
 use SNMP::Info;
 use Try::Tiny;
@@ -65,11 +65,27 @@ sub _snmp_connect_generic {
     DestHost => $device->ip,
     Retries => (setting('snmpretries') || 2),
     Timeout => (setting('snmptimeout') || 1000000),
+    NonIncreasing => (setting('nonincreasing') || 0),
+    BulkWalk => ((defined setting('bulkwalk_off')) ? setting('bulkwalk_off') : 1),
+    BulkRepeaters => (setting('bulkwalk_repeaters') || 20),
     MibDirs => [ _build_mibdirs() ],
     IgnoreNetSNMPConf => 1,
     Debug => ($ENV{INFO_TRACE} || 0),
   );
 
+  # an override for bulkwalk
+  $snmp_args{BulkWalk} = 0 if check_no($device, 'bulkwalk_no');
+
+  # further protect against buggy Net-SNMP, and disable bulkwalk
+  if ($snmp_args{BulkWalk}
+      and ($SNMP::VERSION eq '5.0203' || $SNMP::VERSION eq '5.0301')) {
+
+      warning sprintf
+        "[%s] turning off BulkWalk due to buggy Net-SNMP - please upgrade!",
+        $device->ip;
+      $snmp_args{BulkWalk} = 0;
+  }
+
   # TODO: add version force support
   # use existing SNMP version or try 2, 1
   my @versions = (($device->snmp_ver || setting('snmpver') || 2));
@@ -175,7 +191,7 @@ sub _try_connect {
 
 sub _build_mibdirs {
   my $home = (setting('mibhome') || dir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs'));
-  return map { dir($home, $_) }
+  return map { dir($home, $_)->stringify }
              @{ setting('mibdirs') || _get_mibdirs_content($home) };
 }