135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

async dns support

Browse Source
This commit is contained in:
Eric A. Miller
2013-10-15 22:55:44 -04:00
parent 325e59bade
commit 8946cf291b
10 changed files with 97 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Netdisco/Makefile.PL
View File

@@ -5,6 +5,7 @@ license 'bsd';
all_from 'lib/App/Netdisco.pm';
requires 'Algorithm::Cron' => 0.07;
requires 'AnyEvent' => 7.05;
requires 'App::cpanminus' => 1.6108;
requires 'App::local::lib::helper' => 0.07;
requires 'DBD::Pg' => 0;
@@ -16,6 +17,7 @@ requires 'Dancer' => 1.3112;
requires 'Dancer::Plugin::DBIC' => 0.1803;
requires 'Dancer::Plugin::Auth::Extensible' => 0.20;
requires 'File::ShareDir' => 1.03;
requires 'Guard' => 1.022;
requires 'HTML::Parser' => 3.70;
requires 'HTTP::Tiny' => 0.029;
requires 'JSON' => 0;

73
Netdisco/lib/App/Netdisco/Core/Arpnip.pm
View File

@@ -10,7 +10,7 @@ use Time::HiRes 'gettimeofday';
use base 'Exporter';
our @EXPORT = ();
our @EXPORT_OK = qw/ do_arpnip store_arp resolve_node_names /;
our @EXPORT_OK = qw/ do_arpnip store_arp /;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
=head1 NAME
@@ -44,9 +44,9 @@ sub do_arpnip {
}
# get v4 arp table
my @v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
my $v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
# get v6 neighbor cache
my @v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
my $v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
# get directly connected networks
my @subnets = _gather_subnets($device, $snmp);
@@ -58,13 +58,13 @@ sub do_arpnip {
my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
# update node_ip with ARP and Neighbor Cache entries
store_arp(@$_, $now) for @v4;
store_arp(\%$_, $now) for @$v4;
debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
$device->ip, scalar @v4;
$device->ip, scalar @$v4;
store_arp(@$_, $now) for @v6;
store_arp(\%$_, $now) for @$v6;
debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
$device->ip, scalar @v6;
$device->ip, scalar @$v6;
_store_subnet($_, $now) for @subnets;
debug sprintf ' [%s] arpnip - processed %s Subnet entries',
@@ -82,16 +82,24 @@ sub _get_arps {
my $ip = $netaddr->{$arp};
next unless defined $ip;
next unless check_mac($device, $node);
push @arps, [$node, $ip];
push @arps, {
node => $node,
ip => $ip,
dns => undef,
};
}
return @arps;
debug sprintf ' resolving %d aliases with max %d outstanding requests',
scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
my $resolved_ips = hostnames_resolve_async(\@arps);
return $resolved_ips;
}
=head2 store_arp( $mac, $ip, $now? )
=head2 store_arp( $mac, $ip, $name, $now? )
Stores a new entry to the C<node_ip> table with the given MAC, and IP (v4 or
v6).
Stores a new entry to the C<node_ip> table with the given MAC, IP (v4 or v6)
and DNS host name.
Will mark old entries for this IP as no longer C<active>.
@@ -101,8 +109,11 @@ C<time_last> timestamp, otherwise the current timestamp (C<now()>) is used.
=cut
sub store_arp {
my ($mac, $ip, $now) = @_;
my ($hash_ref, $now) = @_;
$now ||= 'now()';
my $ip = $hash_ref->{'ip'};
my $mac = $hash_ref->{'node'};
my $name = $hash_ref->{'dns'};
schema('netdisco')->txn_do(sub {
my $current = schema('netdisco')->resultset('NodeIp')
@@ -119,6 +130,7 @@ sub store_arp {
->search({'me.mac' => $mac, 'me.ip' => $ip})
->update_or_create(
{
dns => $name,
active => \'true',
time_last => \$now,
},
@@ -171,39 +183,4 @@ sub _store_subnet {
});
}
=head2 resolve_node_names( $device )
Given a Device database object, resolve Node IP (ARP) entries belonging to
this device into DNS names, and store them in the C<node_ip> database table.
This action is usually queued following C<do_arpip> so that it may run
asynchronously, and/or on another daemon worker node.
=cut
sub resolve_node_names {
my ($device) = @_;
schema('netdisco')->txn_do(sub {
my $nodeips = schema('netdisco')
->resultset('NodeIp')->search(
{
-and => [
-bool => 'me.active',
-bool => 'nodes.active',
],
'nodes.switch' => $device->ip,
},
{
join => 'nodes',
for => 'update',
}
);
while (my $nodeip = $nodeips->next) {
$nodeip->update({dns => hostname_from_ip($nodeip->ip)});
}
});
}
1;

12
Netdisco/lib/App/Netdisco/Core/Discover.pm
View File

@@ -76,10 +76,14 @@ sub store_device {
alias => $addr,
port => $port,
subnet => $subnet,
dns => hostname_from_ip($addr),
dns => undef,
};
}
debug sprintf ' resolving %d aliases with max %d outstanding requests',
scalar @aliases, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
my $resolved_aliases = hostnames_resolve_async(\@aliases);
# VTP Management Domain -- assume only one.
my $vtpdomains = $snmp->vtp_d_name;
my $vtpdomain;
@@ -108,7 +112,7 @@ sub store_device {
debug sprintf ' [%s] device - removed %s aliases',
$device->ip, $gone;
$device->update_or_insert(undef, {for => 'update'});
$device->device_ips->populate(\@aliases);
$device->device_ips->populate($resolved_aliases);
debug sprintf ' [%s] device - added %d new aliases',
$device->ip, scalar @aliases;
});
@@ -547,7 +551,7 @@ sub store_modules {
# build device modules list for DBIC
my @modules;
foreach my $entry (keys %$e_class) {
foreach my $entry (keys %$e_index) {
push @modules, {
index => $e_index->{$entry},
type => $e_type->{$entry},
@@ -670,7 +674,7 @@ sub store_neighbors {
if $remote_type !~ /ip phone/i;
}
else {
$remote_type = '';
$remote_type ||= '';
}
# hack for devices seeing multiple neighbors on the port

2
Netdisco/lib/App/Netdisco/Daemon/Queue.pm
View File

@@ -22,7 +22,7 @@ sub capacity_for {
debug "checking local capacity for action $action";
my $action_map = {
Poller => [qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/],
Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck/],
Interactive => [qw/location contact portcontrol portname vlan power/],
};

2
Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm
View File

@@ -13,7 +13,7 @@ my $fqdn = hostfqdn || 'localhost';
my $role_map = {
(map {$_ => 'Poller'}
qw/discoverall discover arpwalk arpnip nodenames macwalk macsuck/),
qw/discoverall discover arpwalk arpnip macwalk macsuck/),
(map {$_ => 'Interactive'}
qw/location contact portcontrol portname vlan power/)
};

49
Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm
View File

@@ -1,17 +1,9 @@
package App::Netdisco::Daemon::Worker::Poller::Arpnip;
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Core::Arpnip 'do_arpnip';
use App::Netdisco::Util::Device qw/get_device is_arpnipable can_nodenames/;
use App::Netdisco::Core::Arpnip 'resolve_node_names';
use App::Netdisco::Daemon::Util ':all';
use NetAddr::IP::Lite ':lower';
use App::Netdisco::Util::Device 'is_arpnipable';
use Role::Tiny;
use Class::Method::Modifiers;
use namespace::clean;
with 'App::Netdisco::Daemon::Worker::Poller::Common';
@@ -23,43 +15,4 @@ sub arpnip_layer { 3 }
sub arpwalk { (shift)->_walk_body('arpnip', @_) }
sub arpnip { (shift)->_single_body('arpnip', @_) }
after 'arpnip' => sub {
my ($self, $job) = @_;
my $host = NetAddr::IP::Lite->new($job->device);
my $device = get_device($host->addr);
my $jobqueue = schema('netdisco')->resultset('Admin');
schema('netdisco')->txn_do(sub {
$jobqueue->create({
device => $device->ip,
action => 'nodenames',
status => 'queued',
username => $job->username,
userip => $job->userip,
});
});
};
# run a nodenames job for one device
sub nodenames {
my ($self, $job) = @_;
my $host = NetAddr::IP::Lite->new($job->device);
my $device = get_device($host->addr);
my $jobqueue = schema('netdisco')->resultset('Admin');
if ($device->ip eq '0.0.0.0') {
return job_error("nodenames failed: no device param (need -d ?)");
}
unless (can_nodenames($device->ip)) {
return job_defer("nodenames deferred: cannot run for $host");
}
resolve_node_names($device);
return job_done("Ended nodenames for ". $host->addr);
}
1;

29
Netdisco/lib/App/Netdisco/Manual/Configuration.pod
View File

@@ -444,22 +444,6 @@ Value: Number. Default: 0.
Sets the minimum amount of time in seconds which must elapse between any two
arpnip jobs for a device.
=head3 C<nodenames_no>
Value: List of Network Identifiers or Device Properties. Default: Empty List.
Devices in the list will not have their Node IPs resolved to names using the
DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in the
list.
=head3 C<nodenames_only>
Value: List of Network Identifiers or Device Properties. Default: Empty List.
Only thos devices in the list will have their Node IPs resolved to names using
the DNS. You can include hostnames, IP addresses and subnets (IPv4 or IPv6) in
the list.
=head3 C<store_wireless_clients>
Value: Boolean. Default: C<true>.
@@ -576,6 +560,19 @@ on this node, respectively. Other nodes can have different settings.
C<sleep_time> is the number of seconds between polling the database to find
new jobs. This is a balance between responsiveness and database load.
=head3 C<dns>
Value: Settings Tree. Default:
dns:
max_outstanding: 250
Sets the maximum number of outstanding requests for asynchronous DNS
resolution used during arpnip and device alias discovery.
This setting overrides the C<PERL_ANYEVENT_MAX_OUTSTANDING_DNS> environment
value and the C<AnyEvent> library default of 10.
=head3 C<housekeeping>
Value: Settings Tree. Default: None.

44
Netdisco/lib/App/Netdisco/Util/DNS.pm
View File

@@ -4,11 +4,12 @@ use strict;
use warnings FATAL => 'all';
use Net::DNS;
use AnyEvent::DNS;
use base 'Exporter';
our @EXPORT = ();
our @EXPORT_OK = qw/
hostname_from_ip ipv4_from_hostname
hostname_from_ip hostnames_resolve_async ipv4_from_hostname
/;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
@@ -75,5 +76,46 @@ sub ipv4_from_hostname {
return undef;
}
=head2 hostnames_resolve_async( $ips )
This method uses a fully asynchronous and high-performance pure-perl stub
resolver C<AnyEvent::DNS>.
Given a reference to an array of hashes will resolve the C<IPv4> or C<IPv6>
address in the C<ip> or C<alias> key of each hash into its hostname which
will be inserted in the C<dns> key of the hash. The resolver does also
forward-lookups to verify that the resolved hostnames point to the
address.
Returns the supplied reference to an array of hashes with dns values for
addresses which resolved.
=cut
sub hostnames_resolve_async {
my $ips = shift;
my $resolver = AnyEvent::DNS->new();
# Set up the condvar
my $done = AE::cv;
$done->begin( sub { shift->send } );
foreach my $hash_ref (@$ips) {
my $ip = $hash_ref->{'ip'} || $hash_ref->{'alias'};
$done->begin;
AnyEvent::DNS::reverse_verify $ip,
sub { $hash_ref->{'dns'} = shift; $done->end; };
}
# Decrement the cv counter to cancel out the send declaration
$done->end;
# Wait for the resolver to perform all resolutions
$done->recv;
return $ips;
}
1;

27
Netdisco/lib/App/Netdisco/Util/Device.pm
View File

@@ -13,7 +13,6 @@ our @EXPORT_OK = qw/
check_only
is_discoverable
is_arpnipable
can_nodenames
is_macsuckable
/;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
@@ -246,32 +245,6 @@ sub is_arpnipable {
return 1;
}
=head2 can_nodenames( $ip )
Given an IP address, returns C<true> if Netdisco on this host is permitted by
the local configuration to resolve Node IPs to DNS names for the device.
The configuration items C<nodenames_no> and C<nodenames_only> are checked
against the given IP.
Returns false if the host is not permitted to do this job for the target
device.
=cut
sub can_nodenames {
my $ip = shift;
my $device = get_device($ip) or return 0;
return _bail_msg("can_nodenames device matched nodenames_no")
if check_no($device, 'nodenames_no');
return _bail_msg("can_nodenames: device failed to match nodenames_only")
unless check_only($device, 'nodenames_only');
return 1;
}
=head2 is_macsuckable( $ip )
Given an IP address, returns C<true> if Netdisco on this host is permitted by

5
Netdisco/share/config.yml
View File

@@ -86,8 +86,6 @@ macsuck_min_age: 0
arpnip_no: []
arpnip_only: []
arpnip_min_age: 0
nodenames_no: []
nodenames_only: []
store_wireless_clients: true
store_modules: true
ignore_interfaces:
@@ -123,6 +121,9 @@ workers:
pollers: 5
sleep_time: 2
dns:
max_outstanding: 250
#housekeeping:
# discoverall:
# when: '0 9 * * *'