From d21ab211307c21a9385a6ef1cb058da1050a8805 Mon Sep 17 00:00:00 2001 From: Christian Ramseyer Date: Sun, 24 Feb 2019 20:18:01 +0100 Subject: [PATCH] Integrate netdisco-sshcollector into Worker::Plugin architecture (#489) * Initial integration of sshcollector into Worker::Plugin architecture * NOT FULLY FUNCTIONAL - this is only to discuss some issues for now * add NodesBySSH.pm * update Build.PL and config.yml to integrate the new module * Further integration of sshcollector into Worker::Plugin architecture * NOT FULLY FUNCTIONAL - this is only to discuss some issues for now * added App::Netdisco::Transport::CLI loosely based on ::SNMP counterpart * switched to the more prevalent two-space tabs style * removed various TBD items, some new ones * Further steps to integration of sshcollector into Worker::Plugin architecture * cleaned up code * added various error handling * warning for bin/netdisco-sshcollector deprecation * device_auth allows passing master_opts to Net::OpenSSH * netdisco-do -D also toggles Net::OpenSSH debug * Merged NodesBySSH.pm into Nodes.pm * see https://github.com/netdisco/netdisco/pull/489#pullrequestreview-205603516 * Further integration of sshcollector into Worker::Plugin architecture * add snmp_arpnip_also option to sshcollector device_auth * cleanup code * Remove big TBD: comment from CLI.pm as doc is updated now --- Build.PL | 4 +- bin/netdisco-sshcollector | 13 +++ lib/App/Netdisco/Transport/CLI.pm | 84 +++++++++++++++++ .../Netdisco/Worker/Plugin/Arpnip/Nodes.pm | 93 +++++++++++++++++-- 4 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 lib/App/Netdisco/Transport/CLI.pm diff --git a/Build.PL b/Build.PL index 8b2de830..0f6807c7 100644 --- a/Build.PL +++ b/Build.PL @@ -37,6 +37,7 @@ Module::Build->new( 'Dancer::Plugin::Auth::Extensible' => '0.30', 'Dancer::Plugin::Passphrase' => '2.0.1', 'Dancer::Session::Cookie' => '0.27', + 'Expect' => '0', 'File::ShareDir' => '1.03', 'File::Slurper' => '0.009', 'Guard' => '1.022', @@ -54,6 +55,7 @@ Module::Build->new( 'Net::Domain' => '1.23', 'Net::DNS' => '0.72', 'Net::LDAP' => '0', + 'Net::OpenSSH' => '0', 'NetAddr::MAC' => '0.93', 'NetAddr::IP' => '4.068', 'Opcode' => '1.07', @@ -90,8 +92,6 @@ Module::Build->new( recommends => { 'Graph' => '0', 'GraphViz' => '0', - 'Net::OpenSSH' => '0', - 'Expect' => '0', }, test_requires => { 'Test::More' => '1.302083', diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 29c0e154..569d06e1 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -78,13 +78,17 @@ if ($opensshdebug){ $Net::OpenSSH::debug = ~0; } + MCE::Loop::init { chunk_size => 1, max_workers => $workers }; my %stats; $stats{entry} = 0; +deprecation_warning(); exit main(); sub main { + + my @input = @{ setting('sshcollector') }; if ($device){ @@ -176,6 +180,15 @@ sub store_arpentries { } } +sub deprecation_warning { + print "\n"; + warning sprintf "DEPRECATION WARNING\n" . + "This script and the sshcollector setting will be removed in a future release!\n". + "See this document to migrate to the new sshcollector integrated into\n" . + "regular netdisco-do/netdisco-daemon arpnip:\n" . + "https://github.com/netdisco/netdisco/wiki/bin-sshcollector-deprecation\n\n"; +} + =head1 NAME netdisco-sshcollector - Collect ARP data for Netdisco from devices without diff --git a/lib/App/Netdisco/Transport/CLI.pm b/lib/App/Netdisco/Transport/CLI.pm new file mode 100644 index 00000000..92592228 --- /dev/null +++ b/lib/App/Netdisco/Transport/CLI.pm @@ -0,0 +1,84 @@ +package App::Netdisco::Transport::CLI; + +use Dancer qw/:syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::Permission ':all'; +use Module::Load (); +use Path::Class 'dir'; +use NetAddr::IP::Lite ':lower'; +use List::Util qw/pairkeys pairfirst/; + +use base 'Dancer::Object::Singleton'; + +=head1 NAME + +App::Netdisco::Transport::CLI + +=head1 DESCRIPTION + +Singleton for CLI connections modelled after L but currently +with minimal functionality. Returns a L instance for a given device IP. Limited +to device_auth stanzas tagged sshcollector. Always returns a new connection which the caller +is supposed to close (or it will be closed when going out of scope) + +=cut + + +sub init { + my ( $class, $self ) = @_; + return $self; +} + +=head1 session_for( $ip, $tag ) + +Given an IP address and a tag, returns an L instance configured for and +connected to that device, as well as the C entry that was chosen for the device. + +Returns C if the connection fails. + +=cut + +sub session_for { + my ($class, $ip, $tag) = @_; + my $device = get_device($ip) or return undef; + + my $device_auth = [grep { $_->{tag} eq $tag } @{setting('device_auth')}]; + + # Currently just the first match is used. Warn if there are more. + my $selected_auth = $device_auth->[0]; + + if (@{$device_auth} > 1){ + warning sprintf " [%s] Transport::CLI - found %d matching entries in device_auth, using the first one", + $device->ip, scalar @{$device_auth}; + } + + my @master_opts = qw(-o BatchMode=no); + push(@master_opts, @{$selected_auth->{ssh_master_opts}}) if $selected_auth->{ssh_master_opts}; + + my $ssh = Net::OpenSSH->new( + $device->ip, + user => $selected_auth->{username}, + password => $selected_auth->{password}, + timeout => 30, + async => 0, + default_stderr_file => '/dev/null', + master_opts => \@master_opts + ); + + my $CONFIG = config(); + $Net::OpenSSH::debug = ~0 if $CONFIG->{log} eq 'debug'; + + if ($ssh->error){ + error sprintf " [%s] Transport::CLI - ssh connection error [%s]", $device->ip, $ssh->error; + return undef; + }elsif (!$ssh){ + error sprintf " [%s] Transport::CLI - Net::OpenSSH instantiation error", $device->ip; + return undef; + }else{ + return ($ssh, $selected_auth); + } +} + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm index 6805737a..75f33c57 100644 --- a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm @@ -3,24 +3,27 @@ package App::Netdisco::Worker::Plugin::Arpnip::Nodes; use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; - +use App::Netdisco::Transport::CLI (); use App::Netdisco::Transport::SNMP (); use App::Netdisco::Util::Node qw/check_mac store_arp/; use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; use Dancer::Plugin::DBIC 'schema'; use Time::HiRes 'gettimeofday'; +use Module::Load (); +use Net::OpenSSH; +use Try::Tiny; register_worker({ phase => 'main', driver => 'snmp' }, sub { my ($job, $workerconf) = @_; my $device = $job->device; my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) - or return Status->defer("arpnip failed: could not SNMP connect to $device"); + or return Status->defer("arpnip snmp failed: could not SNMP connect to $device"); # get v4 arp table - my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); + my $v4 = get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr); # get v6 neighbor cache - my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); + my $v6 = get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); # would be possible just to use now() on updated records, but by using this # same value for them all, we _can_ if we want add a job at the end to @@ -29,19 +32,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { # update node_ip with ARP and Neighbor Cache entries store_arp(\%$_, $now) for @$v4; - debug sprintf ' [%s] arpnip - processed %s ARP Cache entries', + debug sprintf ' [%s] arpnip snmp - processed %s ARP Cache entries', $device->ip, scalar @$v4; store_arp(\%$_, $now) for @$v6; - debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries', + debug sprintf ' [%s] arpnip snmp - processed %s IPv6 Neighbor Cache entries', $device->ip, scalar @$v6; $device->update({last_arpnip => \$now}); - return Status->done("Ended arpnip for $device"); + return Status->done("Ended arpnip snmp for $device"); }); # get an arp table (v4 or v6) -sub get_arps { +sub get_arps_snmp { my ($device, $paddr, $netaddr) = @_; my @arps = (); @@ -63,4 +66,78 @@ sub get_arps { return $resolved_ips; } +register_worker({ phase => 'main', driver => 'cli' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + my ($ssh, $selected_auth) = App::Netdisco::Transport::CLI->session_for($device->ip, "sshcollector"); + + if (get_arps_cli($device, $ssh, $selected_auth)){ + my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; + $device->update({last_arpnip => \$now}); + my $endmsg = "Ended arpnip cli for $device"; + + if ($selected_auth->{'snmp_arpnip_also'}){ + $endmsg .= ", now running arpnip due to snmp_arpnip_also"; + info sprintf " [%s] arpnip cli - $endmsg", $device->ip; + return Status->info($endmsg); + }else{ + info sprintf " [%s] arpnip cli - $endmsg", $device->ip; + return Status->done($endmsg); + } + }else{ + Status->defer("arpnip cli failed"); + } + +}); + +sub get_arps_cli { + my ($device, $ssh, $selected_auth) = @_; + + + unless ($ssh){ + my $msg = "could not connect to $device with SSH, deferring job"; + warning sprintf " [%s] arpnip cli - %s", $device->ip, $msg; + return undef; + } + + my $class = "App::Netdisco::SSHCollector::Platform::".$selected_auth->{platform}; + debug sprintf " [%s] arpnip cli - delegating to platform module %s", $device->ip, $class; + + my $load_failed = 0; + try { + Module::Load::load $class; + } catch { + warning sprintf " [%s] arpnip cli - failed to load %s: %s", $device->ip, $class, substr($_, 0, 50)."..."; + $load_failed = 1; + }; + return undef if $load_failed; + + my $platform_class = $class->new(); + my $arpentries = [ $platform_class->arpnip($device->ip, $ssh, $selected_auth) ]; + + if (not scalar @$arpentries) { + warning sprintf " [%s] WARNING: no entries received from device", $device->ip; + } + + hostnames_resolve_async($arpentries); + + foreach my $arpentry ( @$arpentries ) { + + # skip broadcast/vrrp/hsrp and other weirdos + next unless check_mac( $arpentry->{mac} ); + + debug sprintf ' [%s] arpnip cli - stored entry: %s / %s / %s', + $device->ip, $arpentry->{mac}, $arpentry->{ip}, + $arpentry->{dns} if defined $arpentry->{dns}; + store_arp({ + node => $arpentry->{mac}, + ip => $arpentry->{ip}, + dns => $arpentry->{dns}, + }); + } + + return 1; +} + true;