diff --git a/Netdisco/Makefile.PL b/Netdisco/Makefile.PL index be3ec6a0..03be0a5a 100644 --- a/Netdisco/Makefile.PL +++ b/Netdisco/Makefile.PL @@ -5,6 +5,7 @@ license 'bsd'; all_from 'lib/App/Netdisco.pm'; requires 'Algorithm::Cron' => 0.07; +requires 'AnyEvent' => 7.05; requires 'App::cpanminus' => 1.6108; requires 'App::local::lib::helper' => 0.07; requires 'DBD::Pg' => 0; @@ -16,6 +17,7 @@ requires 'Dancer' => 1.3112; requires 'Dancer::Plugin::DBIC' => 0.1803; requires 'Dancer::Plugin::Auth::Extensible' => 0.20; requires 'File::ShareDir' => 1.03; +requires 'Guard' => 1.022; requires 'HTML::Parser' => 3.70; requires 'HTTP::Tiny' => 0.029; requires 'JSON' => 0; diff --git a/Netdisco/lib/App/Netdisco/Core/Arpnip.pm b/Netdisco/lib/App/Netdisco/Core/Arpnip.pm index 549169bc..ea39ca62 100644 --- a/Netdisco/lib/App/Netdisco/Core/Arpnip.pm +++ b/Netdisco/lib/App/Netdisco/Core/Arpnip.pm @@ -10,7 +10,7 @@ use Time::HiRes 'gettimeofday'; use base 'Exporter'; our @EXPORT = (); -our @EXPORT_OK = qw/ do_arpnip store_arp resolve_node_names /; +our @EXPORT_OK = qw/ do_arpnip store_arp /; our %EXPORT_TAGS = (all => \@EXPORT_OK); =head1 NAME @@ -44,9 +44,9 @@ sub do_arpnip { } # get v4 arp table - my @v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); + my $v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); # get v6 neighbor cache - my @v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); + my $v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); # get directly connected networks my @subnets = _gather_subnets($device, $snmp); @@ -58,13 +58,13 @@ sub do_arpnip { my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; # update node_ip with ARP and Neighbor Cache entries - store_arp(@$_, $now) for @v4; + store_arp(\%$_, $now) for @$v4; debug sprintf ' [%s] arpnip - processed %s ARP Cache entries', - $device->ip, scalar @v4; + $device->ip, scalar @$v4; - store_arp(@$_, $now) for @v6; + store_arp(\%$_, $now) for @$v6; debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries', - $device->ip, scalar @v6; + $device->ip, scalar @$v6; _store_subnet($_, $now) for @subnets; debug sprintf ' [%s] arpnip - processed %s Subnet entries', @@ -82,16 +82,24 @@ sub _get_arps { my $ip = $netaddr->{$arp}; next unless defined $ip; next unless check_mac($device, $node); - push @arps, [$node, $ip]; + push @arps, { + node => $node, + ip => $ip, + dns => undef, + }; } - return @arps; + debug sprintf ' resolving %d aliases with max %d outstanding requests', + scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; + my $resolved_ips = hostnames_resolve_async(\@arps); + + return $resolved_ips; } -=head2 store_arp( $mac, $ip, $now? ) +=head2 store_arp( $mac, $ip, $name, $now? ) -Stores a new entry to the C table with the given MAC, and IP (v4 or -v6). +Stores a new entry to the C table with the given MAC, IP (v4 or v6) +and DNS host name. Will mark old entries for this IP as no longer C. @@ -101,8 +109,11 @@ C timestamp, otherwise the current timestamp (C) is used. =cut sub store_arp { - my ($mac, $ip, $now) = @_; + my ($hash_ref, $now) = @_; $now ||= 'now()'; + my $ip = $hash_ref->{'ip'}; + my $mac = $hash_ref->{'node'}; + my $name = $hash_ref->{'dns'}; schema('netdisco')->txn_do(sub { my $current = schema('netdisco')->resultset('NodeIp') @@ -119,6 +130,7 @@ sub store_arp { ->search({'me.mac' => $mac, 'me.ip' => $ip}) ->update_or_create( { + dns => $name, active => \'true', time_last => \$now, }, @@ -171,39 +183,4 @@ sub _store_subnet { }); } -=head2 resolve_node_names( $device ) - -Given a Device database object, resolve Node IP (ARP) entries belonging to -this device into DNS names, and store them in the C database table. - -This action is usually queued following C so that it may run -asynchronously, and/or on another daemon worker node. - -=cut - -sub resolve_node_names { - my ($device) = @_; - - schema('netdisco')->txn_do(sub { - my $nodeips = schema('netdisco') - ->resultset('NodeIp')->search( - { - -and => [ - -bool => 'me.active', - -bool => 'nodes.active', - ], - 'nodes.switch' => $device->ip, - }, - { - join => 'nodes', - for => 'update', - } - ); - - while (my $nodeip = $nodeips->next) { - $nodeip->update({dns => hostname_from_ip($nodeip->ip)}); - } - }); -} - 1; diff --git a/Netdisco/lib/App/Netdisco/Core/Discover.pm b/Netdisco/lib/App/Netdisco/Core/Discover.pm index 6371f55b..6f8a8f99 100644 --- a/Netdisco/lib/App/Netdisco/Core/Discover.pm +++ b/Netdisco/lib/App/Netdisco/Core/Discover.pm @@ -76,10 +76,14 @@ sub store_device { alias => $addr, port => $port, subnet => $subnet, - dns => hostname_from_ip($addr), + dns => undef, }; } + debug sprintf ' resolving %d aliases with max %d outstanding requests', + scalar @aliases, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; + my $resolved_aliases = hostnames_resolve_async(\@aliases); + # VTP Management Domain -- assume only one. my $vtpdomains = $snmp->vtp_d_name; my $vtpdomain; @@ -108,7 +112,7 @@ sub store_device { debug sprintf ' [%s] device - removed %s aliases', $device->ip, $gone; $device->update_or_insert(undef, {for => 'update'}); - $device->device_ips->populate(\@aliases); + $device->device_ips->populate($resolved_aliases); debug sprintf ' [%s] device - added %d new aliases', $device->ip, scalar @aliases; }); @@ -547,7 +551,7 @@ sub store_modules { # build device modules list for DBIC my @modules; - foreach my $entry (keys %$e_class) { + foreach my $entry (keys %$e_index) { push @modules, { index => $e_index->{$entry}, type => $e_type->{$entry}, @@ -670,7 +674,7 @@ sub store_neighbors { if $remote_type !~ /ip phone/i; } else { - $remote_type = ''; + $remote_type ||= ''; } # hack for devices seeing multiple neighbors on the port diff --git a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm index 336f65a4..f89e14fb 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm @@ -22,7 +22,7 @@ sub capacity_for { debug "checking local capacity for action $action"; my $action_map = { - Poller => [qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/], + Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck/], Interactive => [qw/location contact portcontrol portname vlan power/], }; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index 0087d149..047196f2 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -13,7 +13,7 @@ my $fqdn = hostfqdn || 'localhost'; my $role_map = { (map {$_ => 'Poller'} - qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/), + qw/discoverall discover arpwalk arpnip macwalk macsuck/), (map {$_ => 'Interactive'} qw/location contact portcontrol portname vlan power/) }; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm index 26e47be2..d511c1b4 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm @@ -1,17 +1,9 @@ package App::Netdisco::Daemon::Worker::Poller::Arpnip; -use Dancer::Plugin::DBIC 'schema'; - use App::Netdisco::Core::Arpnip 'do_arpnip'; -use App::Netdisco::Util::Device qw/get_device is_arpnipable can_nodenames/; - -use App::Netdisco::Core::Arpnip 'resolve_node_names'; -use App::Netdisco::Daemon::Util ':all'; - -use NetAddr::IP::Lite ':lower'; +use App::Netdisco::Util::Device 'is_arpnipable'; use Role::Tiny; -use Class::Method::Modifiers; use namespace::clean; with 'App::Netdisco::Daemon::Worker::Poller::Common'; @@ -23,43 +15,4 @@ sub arpnip_layer { 3 } sub arpwalk { (shift)->_walk_body('arpnip', @_) } sub arpnip { (shift)->_single_body('arpnip', @_) } -after 'arpnip' => sub { - my ($self, $job) = @_; - - my $host = NetAddr::IP::Lite->new($job->device); - my $device = get_device($host->addr); - my $jobqueue = schema('netdisco')->resultset('Admin'); - - schema('netdisco')->txn_do(sub { - $jobqueue->create({ - device => $device->ip, - action => 'nodenames', - status => 'queued', - username => $job->username, - userip => $job->userip, - }); - }); -}; - -# run a nodenames job for one device -sub nodenames { - my ($self, $job) = @_; - - my $host = NetAddr::IP::Lite->new($job->device); - my $device = get_device($host->addr); - my $jobqueue = schema('netdisco')->resultset('Admin'); - - if ($device->ip eq '0.0.0.0') { - return job_error("nodenames failed: no device param (need -d ?)"); - } - - unless (can_nodenames($device->ip)) { - return job_defer("nodenames deferred: cannot run for $host"); - } - - resolve_node_names($device); - - return job_done("Ended nodenames for ". $host->addr); -} - 1; diff --git a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod index 906f0aa2..c2700aca 100644 --- a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod +++ b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod @@ -444,22 +444,6 @@ Value: Number. Default: 0. Sets the minimum amount of time in seconds which must elapse between any two arpnip jobs for a device. -=head3 C - -Value: List of Network Identifiers or Device Properties. Default: Empty List. - -Devices in the list will not have their Node IPs resolved to names using the -DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in the -list. - -=head3 C - -Value: List of Network Identifiers or Device Properties. Default: Empty List. - -Only thos devices in the list will have their Node IPs resolved to names using -the DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in -the list. - =head3 C Value: Boolean. Default: C. @@ -576,6 +560,19 @@ on this node, respectively. Other nodes can have different settings. C is the number of seconds between polling the database to find new jobs. This is a balance between responsiveness and database load. +=head3 C + +Value: Settings Tree. Default: + + dns: + max_outstanding: 250 + +Sets the maximum number of outstanding requests for asynchronous DNS +resolution used during arpnip and device alias discovery. + +This setting overrides the C environment +value and the C library default of 10. + =head3 C Value: Settings Tree. Default: None. diff --git a/Netdisco/lib/App/Netdisco/Util/DNS.pm b/Netdisco/lib/App/Netdisco/Util/DNS.pm index 58377cc3..92d8e6cf 100644 --- a/Netdisco/lib/App/Netdisco/Util/DNS.pm +++ b/Netdisco/lib/App/Netdisco/Util/DNS.pm @@ -4,11 +4,12 @@ use strict; use warnings FATAL => 'all'; use Net::DNS; +use AnyEvent::DNS; use base 'Exporter'; our @EXPORT = (); our @EXPORT_OK = qw/ - hostname_from_ip ipv4_from_hostname + hostname_from_ip hostnames_resolve_async ipv4_from_hostname /; our %EXPORT_TAGS = (all => \@EXPORT_OK); @@ -75,5 +76,46 @@ sub ipv4_from_hostname { return undef; } +=head2 hostnames_resolve_async( $ips ) + +This method uses a fully asynchronous and high-performance pure-perl stub +resolver C. + +Given a reference to an array of hashes will resolve the C or C +address in the C or C key of each hash into its hostname which +will be inserted in the C key of the hash. The resolver does also +forward-lookups to verify that the resolved hostnames point to the +address. + +Returns the supplied reference to an array of hashes with dns values for +addresses which resolved. + +=cut + +sub hostnames_resolve_async { + my $ips = shift; + + my $resolver = AnyEvent::DNS->new(); + + # Set up the condvar + my $done = AE::cv; + $done->begin( sub { shift->send } ); + + foreach my $hash_ref (@$ips) { + my $ip = $hash_ref->{'ip'} || $hash_ref->{'alias'}; + $done->begin; + AnyEvent::DNS::reverse_verify $ip, + sub { $hash_ref->{'dns'} = shift; $done->end; }; + } + + # Decrement the cv counter to cancel out the send declaration + $done->end; + + # Wait for the resolver to perform all resolutions + $done->recv; + + return $ips; +} + 1; diff --git a/Netdisco/lib/App/Netdisco/Util/Device.pm b/Netdisco/lib/App/Netdisco/Util/Device.pm index 11863c71..4b31c754 100644 --- a/Netdisco/lib/App/Netdisco/Util/Device.pm +++ b/Netdisco/lib/App/Netdisco/Util/Device.pm @@ -13,7 +13,6 @@ our @EXPORT_OK = qw/ check_only is_discoverable is_arpnipable - can_nodenames is_macsuckable /; our %EXPORT_TAGS = (all => \@EXPORT_OK); @@ -246,32 +245,6 @@ sub is_arpnipable { return 1; } -=head2 can_nodenames( $ip ) - -Given an IP address, returns C if Netdisco on this host is permitted by -the local configuration to resolve Node IPs to DNS names for the device. - -The configuration items C and C are checked -against the given IP. - -Returns false if the host is not permitted to do this job for the target -device. - -=cut - -sub can_nodenames { - my $ip = shift; - my $device = get_device($ip) or return 0; - - return _bail_msg("can_nodenames device matched nodenames_no") - if check_no($device, 'nodenames_no'); - - return _bail_msg("can_nodenames: device failed to match nodenames_only") - unless check_only($device, 'nodenames_only'); - - return 1; -} - =head2 is_macsuckable( $ip ) Given an IP address, returns C if Netdisco on this host is permitted by diff --git a/Netdisco/share/config.yml b/Netdisco/share/config.yml index f8a366e8..da41bd0f 100644 --- a/Netdisco/share/config.yml +++ b/Netdisco/share/config.yml @@ -86,8 +86,6 @@ macsuck_min_age: 0 arpnip_no: [] arpnip_only: [] arpnip_min_age: 0 -nodenames_no: [] -nodenames_only: [] store_wireless_clients: true store_modules: true ignore_interfaces: @@ -123,6 +121,9 @@ workers: pollers: 5 sleep_time: 2 +dns: + max_outstanding: 250 + #housekeeping: # discoverall: # when: '0 9 * * *'