async dns support

Netdisco/Makefile.PL

@@ -5,6 +5,7 @@ license  'bsd';
 all_from 'lib/App/Netdisco.pm';
  
 requires 'Algorithm::Cron' => 0.07;
+requires 'AnyEvent' => 7.05;
 requires 'App::cpanminus' => 1.6108;
 requires 'App::local::lib::helper' => 0.07;
 requires 'DBD::Pg' => 0;
@@ -16,6 +17,7 @@ requires 'Dancer' => 1.3112;
 requires 'Dancer::Plugin::DBIC' => 0.1803;
 requires 'Dancer::Plugin::Auth::Extensible' => 0.20;
 requires 'File::ShareDir' => 1.03;
+requires 'Guard' => 1.022;
 requires 'HTML::Parser' => 3.70;
 requires 'HTTP::Tiny' => 0.029;
 requires 'JSON' => 0;

Netdisco/lib/App/Netdisco/Core/Arpnip.pm

@@ -10,7 +10,7 @@ use Time::HiRes 'gettimeofday';
 
 use base 'Exporter';
 our @EXPORT = ();
-our @EXPORT_OK = qw/ do_arpnip store_arp resolve_node_names /;
+our @EXPORT_OK = qw/ do_arpnip store_arp /;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
 =head1 NAME
@@ -44,9 +44,9 @@ sub do_arpnip {
   }
 
   # get v4 arp table
-  my @v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
+  my $v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
   # get v6 neighbor cache
-  my @v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
+  my $v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
 
   # get directly connected networks
   my @subnets = _gather_subnets($device, $snmp);
@@ -58,13 +58,13 @@ sub do_arpnip {
   my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
 
   # update node_ip with ARP and Neighbor Cache entries
-  store_arp(@$_, $now) for @v4;
+  store_arp(\%$_, $now) for @$v4;
   debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
-    $device->ip, scalar @v4;
+    $device->ip, scalar @$v4;
 
-  store_arp(@$_, $now) for @v6;
+  store_arp(\%$_, $now) for @$v6;
   debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
-    $device->ip, scalar @v6;
+    $device->ip, scalar @$v6;
 
   _store_subnet($_, $now) for @subnets;
   debug sprintf ' [%s] arpnip - processed %s Subnet entries',
@@ -82,16 +82,24 @@ sub _get_arps {
       my $ip = $netaddr->{$arp};
       next unless defined $ip;
       next unless check_mac($device, $node);
-      push @arps, [$node, $ip];
+      push @arps, {
+        node => $node,
+        ip   => $ip,
+        dns  => undef,
+      };
   }
 
-  return @arps;
+  debug sprintf ' resolving %d aliases with max %d outstanding requests',
+      scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_ips = hostnames_resolve_async(\@arps);
+
+  return $resolved_ips;
 }
 
-=head2 store_arp( $mac, $ip, $now? )
+=head2 store_arp( $mac, $ip, $name, $now? )
 
-Stores a new entry to the C<node_ip> table with the given MAC, and IP (v4 or
+Stores a new entry to the C<node_ip> table with the given MAC, IP (v4 or v6)
-v6).
+and DNS host name.
 
 Will mark old entries for this IP as no longer C<active>.
 
@@ -101,8 +109,11 @@ C<time_last> timestamp, otherwise the current timestamp (C<now()>) is used.
 =cut
 
 sub store_arp {
-  my ($mac, $ip, $now) = @_;
+  my ($hash_ref, $now) = @_;
   $now ||= 'now()';
+  my $ip   = $hash_ref->{'ip'};
+  my $mac  = $hash_ref->{'node'};
+  my $name = $hash_ref->{'dns'};
 
   schema('netdisco')->txn_do(sub {
     my $current = schema('netdisco')->resultset('NodeIp')
@@ -119,6 +130,7 @@ sub store_arp {
       ->search({'me.mac' => $mac, 'me.ip' => $ip})
       ->update_or_create(
       {
+        dns => $name,
         active => \'true',
         time_last => \$now,
       },
@@ -171,39 +183,4 @@ sub _store_subnet {
   });
 }
 
-=head2 resolve_node_names( $device )
-
-Given a Device database object, resolve Node IP (ARP) entries belonging to
-this device into DNS names, and store them in the C<node_ip> database table.
-
-This action is usually queued following C<do_arpip> so that it may run
-asynchronously, and/or on another daemon worker node.
-
-=cut
-
-sub resolve_node_names {
-  my ($device) = @_;
-
-  schema('netdisco')->txn_do(sub {
-    my $nodeips = schema('netdisco')
-      ->resultset('NodeIp')->search(
-        {
-          -and => [
-            -bool => 'me.active',
-            -bool => 'nodes.active',
-          ],
-          'nodes.switch' => $device->ip,
-        },
-        {
-          join => 'nodes',
-          for => 'update',
-        }
-      );
-
-    while (my $nodeip = $nodeips->next) {
-      $nodeip->update({dns => hostname_from_ip($nodeip->ip)});
-    }
-  });
-}
-
 1;

Netdisco/lib/App/Netdisco/Core/Discover.pm

@@ -76,10 +76,14 @@ sub store_device {
           alias => $addr,
           port => $port,
           subnet => $subnet,
-          dns => hostname_from_ip($addr),
+          dns => undef,
       };
   }
 
+  debug sprintf ' resolving %d aliases with max %d outstanding requests',
+      scalar @aliases, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_aliases = hostnames_resolve_async(\@aliases);
+
   # VTP Management Domain -- assume only one.
   my $vtpdomains = $snmp->vtp_d_name;
   my $vtpdomain;
@@ -108,7 +112,7 @@ sub store_device {
     debug sprintf ' [%s] device - removed %s aliases',
       $device->ip, $gone;
     $device->update_or_insert(undef, {for => 'update'});
-    $device->device_ips->populate(\@aliases);
+    $device->device_ips->populate($resolved_aliases);
     debug sprintf ' [%s] device - added %d new aliases',
       $device->ip, scalar @aliases;
   });
@@ -547,7 +551,7 @@ sub store_modules {
 
   # build device modules list for DBIC
   my @modules;
-  foreach my $entry (keys %$e_class) {
+  foreach my $entry (keys %$e_index) {
       push @modules, {
           index  => $e_index->{$entry},
           type   => $e_type->{$entry},
@@ -670,7 +674,7 @@ sub store_neighbors {
             if $remote_type !~ /ip phone/i;
       }
       else {
-          $remote_type = '';
+          $remote_type ||= '';
       }
 
       # hack for devices seeing multiple neighbors on the port

Netdisco/lib/App/Netdisco/Daemon/Queue.pm

@@ -22,7 +22,7 @@ sub capacity_for {
   debug "checking local capacity for action $action";
 
   my $action_map = {
-    Poller => [qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/],
+    Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck/],
     Interactive => [qw/location contact portcontrol portname vlan power/],
   };
 

Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm

@@ -13,7 +13,7 @@ my $fqdn = hostfqdn || 'localhost';
 
 my $role_map = {
   (map {$_ => 'Poller'}
-      qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/),
+      qw/discoverall discover arpwalk arpnip macwalk macsuck/),
   (map {$_ => 'Interactive'}
       qw/location contact portcontrol portname vlan power/)
 };

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm

@@ -1,17 +1,9 @@
 package App::Netdisco::Daemon::Worker::Poller::Arpnip;
 
-use Dancer::Plugin::DBIC 'schema';
-
 use App::Netdisco::Core::Arpnip 'do_arpnip';
-use App::Netdisco::Util::Device qw/get_device is_arpnipable can_nodenames/;
+use App::Netdisco::Util::Device 'is_arpnipable';
-
-use App::Netdisco::Core::Arpnip 'resolve_node_names';
-use App::Netdisco::Daemon::Util ':all';
-
-use NetAddr::IP::Lite ':lower';
 
 use Role::Tiny;
-use Class::Method::Modifiers;
 use namespace::clean;
 
 with 'App::Netdisco::Daemon::Worker::Poller::Common';
@@ -23,43 +15,4 @@ sub arpnip_layer { 3 }
 sub arpwalk { (shift)->_walk_body('arpnip', @_) }
 sub arpnip  { (shift)->_single_body('arpnip', @_) }
 
-after 'arpnip' => sub {
-  my ($self, $job) = @_;
-
-  my $host = NetAddr::IP::Lite->new($job->device);
-  my $device = get_device($host->addr);
-  my $jobqueue = schema('netdisco')->resultset('Admin');
-
-  schema('netdisco')->txn_do(sub {
-    $jobqueue->create({
-      device => $device->ip,
-      action => 'nodenames',
-      status => 'queued',
-      username => $job->username,
-      userip => $job->userip,
-    });
-  });
-};
-
-# run a nodenames job for one device
-sub nodenames {
-  my ($self, $job) = @_;
-
-  my $host = NetAddr::IP::Lite->new($job->device);
-  my $device = get_device($host->addr);
-  my $jobqueue = schema('netdisco')->resultset('Admin');
-
-  if ($device->ip eq '0.0.0.0') {
-      return job_error("nodenames failed: no device param (need -d ?)");
-  }
-
-  unless (can_nodenames($device->ip)) {
-      return job_defer("nodenames deferred: cannot run for $host");
-  }
-
-  resolve_node_names($device);
-
-  return job_done("Ended nodenames for ". $host->addr);
-}
-
 1;

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -444,22 +444,6 @@ Value: Number. Default: 0.
 Sets the minimum amount of time in seconds which must elapse between any two
 arpnip jobs for a device.
 
-=head3 C<nodenames_no>
-
-Value: List of Network Identifiers or Device Properties. Default: Empty List.
-
-Devices in the list will not have their Node IPs resolved to names using the
-DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in the
-list.
-
-=head3 C<nodenames_only>
-
-Value: List of Network Identifiers or Device Properties. Default: Empty List.
-
-Only thos devices in the list will have their Node IPs resolved to names using
-the DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in
-the list.
-
 =head3 C<store_wireless_clients>
 
 Value: Boolean. Default: C<true>.
@@ -576,6 +560,19 @@ on this node, respectively. Other nodes can have different settings.
 C<sleep_time> is the number of seconds between polling the database to find
 new jobs. This is a balance between responsiveness and database load.
 
+=head3 C<dns>
+
+Value: Settings Tree. Default:
+
+ dns:
+   max_outstanding: 250
+
+Sets the maximum number of outstanding requests for asynchronous DNS
+resolution used during arpnip and device alias discovery.
+
+This setting overrides the C<PERL_ANYEVENT_MAX_OUTSTANDING_DNS> environment
+value and the C<AnyEvent> library default of 10.  
+
 =head3 C<housekeeping>
 
 Value: Settings Tree. Default: None.

Netdisco/lib/App/Netdisco/Util/DNS.pm

@@ -4,11 +4,12 @@ use strict;
 use warnings FATAL => 'all';
 
 use Net::DNS;
+use AnyEvent::DNS;
 
 use base 'Exporter';
 our @EXPORT = ();
 our @EXPORT_OK = qw/
-  hostname_from_ip ipv4_from_hostname
+  hostname_from_ip hostnames_resolve_async ipv4_from_hostname
 /;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
@@ -75,5 +76,46 @@ sub ipv4_from_hostname {
   return undef;
 }
 
+=head2 hostnames_resolve_async( $ips )
+
+This method uses a fully asynchronous and high-performance pure-perl stub
+resolver C<AnyEvent::DNS>.
+
+Given a reference to an array of hashes will resolve the C<IPv4> or C<IPv6>
+address in the C<ip> or C<alias> key of each hash into its hostname which
+will be inserted in the C<dns> key of the hash.  The resolver does also
+forward-lookups to verify that the resolved hostnames point to the
+address.
+
+Returns the supplied reference to an array of hashes with dns values for
+addresses which resolved.
+
+=cut
+
+sub hostnames_resolve_async {
+  my $ips = shift;
+
+  my $resolver = AnyEvent::DNS->new();
+    
+  # Set up the condvar
+  my $done = AE::cv;
+  $done->begin( sub { shift->send } );
+
+  foreach my $hash_ref (@$ips) {
+    my $ip = $hash_ref->{'ip'} || $hash_ref->{'alias'};
+    $done->begin;
+    AnyEvent::DNS::reverse_verify $ip,
+            sub { $hash_ref->{'dns'} = shift; $done->end; };
+  }
+
+  # Decrement the cv counter to cancel out the send declaration
+  $done->end;
+
+  # Wait for the resolver to perform all resolutions
+  $done->recv;
+
+  return $ips;
+}
+
 1;
 

Netdisco/lib/App/Netdisco/Util/Device.pm

@@ -13,7 +13,6 @@ our @EXPORT_OK = qw/
   check_only
   is_discoverable
   is_arpnipable
-  can_nodenames
   is_macsuckable
 /;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
@@ -246,32 +245,6 @@ sub is_arpnipable {
   return 1;
 }
 
-=head2 can_nodenames( $ip )
-
-Given an IP address, returns C<true> if Netdisco on this host is permitted by
-the local configuration to resolve Node IPs to DNS names for the device.
-
-The configuration items C<nodenames_no> and C<nodenames_only> are checked
-against the given IP.
-
-Returns false if the host is not permitted to do this job for the target
-device.
-
-=cut
-
-sub can_nodenames {
-  my $ip = shift;
-  my $device = get_device($ip) or return 0;
-
-  return _bail_msg("can_nodenames device matched nodenames_no")
-    if check_no($device, 'nodenames_no');
-
-  return _bail_msg("can_nodenames: device failed to match nodenames_only")
-    unless check_only($device, 'nodenames_only');
-
-  return 1;
-}
-
 =head2 is_macsuckable( $ip )
 
 Given an IP address, returns C<true> if Netdisco on this host is permitted by

Netdisco/share/config.yml

@@ -86,8 +86,6 @@ macsuck_min_age: 0
 arpnip_no: []
 arpnip_only: []
 arpnip_min_age: 0
-nodenames_no: []
-nodenames_only: []
 store_wireless_clients: true
 store_modules: true
 ignore_interfaces:
@@ -123,6 +121,9 @@ workers:
   pollers: 5
   sleep_time: 2
 
+dns:
+  max_outstanding: 250
+
 #housekeeping:
 #  discoverall:
 #    when: '0 9 * * *'