From d44a8f56ea25496d460fb7708dd5436dd61abcb8 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 12 Mar 2019 18:50:24 +0000 Subject: [PATCH] Integrate netdisco-sshcollector into Worker::Plugin architecture (#489) (#535) * update changes and SNMP::Info dep * Integrate netdisco-sshcollector into Worker::Plugin architecture (#489) * Initial integration of sshcollector into Worker::Plugin architecture * add NodesBySSH.pm * update Build.PL and config.yml to integrate the new module * Further integration of sshcollector into Worker::Plugin architecture * added App::Netdisco::Transport::CLI loosely based on ::SNMP counterpart * switched to the more prevalent two-space tabs style * removed various TBD items, some new ones * Further steps to integration of sshcollector into Worker::Plugin architecture * cleaned up code * added various error handling * warning for bin/netdisco-sshcollector deprecation * device_auth allows passing master_opts to Net::OpenSSH * netdisco-do -D also toggles Net::OpenSSH debug * Merged NodesBySSH.pm into Nodes.pm * see https://github.com/netdisco/netdisco/pull/489#pullrequestreview-205603516 * Further integration of sshcollector into Worker::Plugin architecture * add snmp_arpnip_also option to sshcollector device_auth * cleanup code * Remove big TBD: comment from CLI.pm as doc is updated now * add transport/cli.pm to manifest * revert some changes to allow simpler merging * silent exit legacy script unless explicitly requested * move ssh code into Transport, part one * rewrite the CLI transport to provide an API * merge in og-get_external_credentials Squashed commit of the following: commit 3fe8f383a791055f8ef4d463d25af6640730ceef Author: Oliver Gorwits Date: Mon Mar 11 17:07:42 2019 +0000 add debug lines and tested commit 3249739e4205eb35ab5841f31b05a4ed34d55475 Author: Oliver Gorwits Date: Mon Mar 11 16:54:11 2019 +0000 change config name to get_credentials commit e78558397abf4b825f6b0a943e630133060565f2 Author: Oliver Gorwits Date: Mon Mar 11 16:51:11 2019 +0000 separate out generic device auth to DeviceAuth module commit 249f05165fd25bdd43802a95c523f0486fc013de Author: Oliver Gorwits Date: Wed Mar 6 18:43:31 2019 +0000 release 2.040007 commit e3af64df77613b98c5efde9d5e7381ab0f5de52d Author: Oliver Gorwits Date: Wed Mar 6 18:42:47 2019 +0000 #521-redux fix wifi date search commit 48857ae3002536b469acb678292981377b960b65 Author: Oliver Gorwits Date: Mon Mar 4 12:03:31 2019 +0000 release 2.040006 commit e09dab53622960bee715d9d423ed2dee91b2e36c Author: Oliver Gorwits Date: Mon Mar 4 11:39:12 2019 +0000 #527 update List::MoreUtils version requirement commit 6e7de3fff3719720da72db76801c9e2e45b5be78 Author: Oliver Gorwits Date: Mon Mar 4 09:59:41 2019 +0000 release 2.040005 commit 0c98318a4508d9c91a8d1182b38e556454f9c9fa Author: Oliver Gorwits Date: Mon Mar 4 09:57:18 2019 +0000 #526 fix discover syntax bug commit e9efc45182f590ea94ddd2928e8156162d022b0d Author: Oliver Gorwits Date: Sun Mar 3 14:56:48 2019 +0000 release 2.040004 commit 6cdfd80d10975c22a9aa2050a02e2ed45dbad3c6 Author: Oliver Gorwits Date: Sun Mar 3 14:34:00 2019 +0000 allow undiscovered neighbors report to use discover_{waps,phones} setting commit ac381e080235b2efc0878b4ccfaa5c2d6901b2ff Author: Oliver Gorwits Date: Sun Mar 3 14:13:20 2019 +0000 #506 was a red herring commit b83e614c851beafd68d131999698023c5380338c Author: Oliver Gorwits Date: Sun Mar 3 13:00:36 2019 +0000 make discover_{phones,waps} work with LLDP capabilities as well commit 189d234b55c01b8e7def428bed9c4be35022ea59 Author: Oliver Gorwits Date: Sun Mar 3 12:47:38 2019 +0000 check discover_no_type and friends earlier on in neighbors list build commit 9c956466f3bf2e799412fef5b61966fb2878bc49 Author: Oliver Gorwits Date: Sun Mar 3 12:32:07 2019 +0000 also update default config for new discover_phones and discover_waps settings commit 09d29954d211ad1e6c1911fb4d43df8980a00bae Author: Oliver Gorwits Date: Sun Mar 3 12:26:50 2019 +0000 #512 fix regression in phone/wap discovery exclusion commit 2bae91f1b669e2901ebcb59696fdaa562c0396b7 Author: Oliver Gorwits Date: Sun Mar 3 12:01:34 2019 +0000 rename match_devicetype() to match_to_setting() commit 57cb6ddb707b938b8a5c0d39a52b9e4c8a8892da Author: Oliver Gorwits Date: Sun Mar 3 09:19:39 2019 +0000 fix for over-eager fix to #506 commit ef560fb59aebf4f7cf64487931d41b6f5aa83d92 Author: Oliver Gorwits Date: Sat Mar 2 22:41:40 2019 +0000 #506 relax device renumber so it works for an alias commit 7a8bcb094e4d48dd0d900d275ca5ce781ddb42ba Author: Oliver Gorwits Date: Sat Mar 2 22:23:39 2019 +0000 #521 Search Node Date Range not working commit a643820a6200a7439724158a6817f3858732050c Author: Oliver Gorwits Date: Sat Mar 2 21:54:27 2019 +0000 #428 Port-Channels not showing in netmap commit 5ba5bcd295230cc77be42255836c380e99e54278 Merge: e7aacddb a1f95028 Author: Oliver Gorwits Date: Sat Mar 2 20:04:11 2019 +0000 Merge branch 'master' of github.com:netdisco/netdisco commit e7aacddbc615a48e5dd204b9dc32a5deadbdefea Author: Oliver Gorwits Date: Sat Mar 2 20:01:05 2019 +0000 #498 Map with VLAN filter omits unconnected devices commit a1f95028caf3d59133dac0bb43fd9d26c44a9dfb Author: nick n <39005454+inphobia@users.noreply.github.com> Date: Sat Mar 2 19:54:22 2019 +0100 catch up with changes noticed that rc-sshcollector-core received updates to changes, add them here as well. didn't mention #499 & #522 commit ce1b847ceabb3cb8ec6bac370071d55acd40dc48 Author: Oliver Gorwits Date: Sat Mar 2 18:47:44 2019 +0000 fix bug showing no nodes when only one matches in netmap commit 78e30a7926dbec941860420348c917ae823146f3 Author: Oliver Gorwits Date: Sat Mar 2 16:28:15 2019 +0000 #500 filtering in device/ports on native vlan duplicates entries commit 9952f0c6c723359c7c1e29abfc0dcad30a52c083 Author: Oliver Gorwits Date: Sat Mar 2 15:02:12 2019 +0000 #499 netdisco-do renumber reports wrong ip (inphobia) commit ca3fd8f466f144f76a9df6d760a5d8f34ae14621 Author: Oliver Gorwits Date: Sat Mar 2 15:00:18 2019 +0000 #505 device renumber should update device port properties and device skips commit 1265bc8470911d90c7c446c64976207c3285a151 Author: Oliver Gorwits Date: Sat Mar 2 14:52:21 2019 +0000 #520 catch slave ports defined without a master commit d4c7579c101d693827da2f3fad533ad725f97ba0 Author: Oliver Gorwits Date: Sat Mar 2 14:47:49 2019 +0000 #522 TypeAhead.pm can reference empty data (inphobia) commit 77decc23b73cbcda0cd0516fdc73972cda622a0b Author: Oliver Gorwits Date: Sat Mar 2 14:45:37 2019 +0000 #514 inconsistent results in ip inventory (inphobia) commit 3f211650b89e58c80d862632faf6c9529c6f8b27 Author: nick n <39005454+inphobia@users.noreply.github.com> Date: Fri Mar 1 12:34:42 2019 +0100 last pieces for db schema upgrade last piece of #510 * import legacy sshcollector config * add default use_legacy_sshcollector config * remove unneeded deps * various fixes and now tested * enable sshcollector platform tests --- Build.PL | 4 +- bin/netdisco-sshcollector | 3 + lib/App/Netdisco/Configuration.pm | 1 + lib/App/Netdisco/Transport/CLI.pm | 115 ++++++++++++++++++ lib/App/Netdisco/Transport/SNMP.pm | 2 +- lib/App/Netdisco/Util/DeviceAuth.pm | 18 ++- .../Netdisco/Worker/Plugin/Arpnip/Nodes.pm | 49 +++++++- share/config.yml | 1 + xt/00-compile.t | 4 +- 9 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 lib/App/Netdisco/Transport/CLI.pm diff --git a/Build.PL b/Build.PL index fe0e4bba..9f20ce47 100644 --- a/Build.PL +++ b/Build.PL @@ -37,6 +37,7 @@ Module::Build->new( 'Dancer::Plugin::Auth::Extensible' => '0.30', 'Dancer::Plugin::Passphrase' => '2.0.1', 'Dancer::Session::Cookie' => '0.27', + 'Expect' => '0', 'File::ShareDir' => '1.03', 'File::Slurper' => '0.009', 'Guard' => '1.022', @@ -54,6 +55,7 @@ Module::Build->new( 'Net::Domain' => '1.23', 'Net::DNS' => '0.72', 'Net::LDAP' => '0', + 'Net::OpenSSH' => '0', 'NetAddr::MAC' => '0.93', 'NetAddr::IP' => '4.068', 'Opcode' => '1.07', @@ -90,8 +92,6 @@ Module::Build->new( recommends => { 'Graph' => '0', 'GraphViz' => '0', - 'Net::OpenSSH' => '0', - 'Expect' => '0', }, test_requires => { 'Test::More' => '1.302083', diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 29c0e154..0f42f7c0 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -74,6 +74,9 @@ $ENV{DBIC_TRACE} ||= $sqltrace; # reconfigure logging to force console output Dancer::Logger->init('console', $CONFIG); +# silent exit unless explicitly requested +exit(0) unless setting('use_legacy_sshcollector'); + if ($opensshdebug){ $Net::OpenSSH::debug = ~0; } diff --git a/lib/App/Netdisco/Configuration.pm b/lib/App/Netdisco/Configuration.pm index 9ddd1d0f..cf076069 100644 --- a/lib/App/Netdisco/Configuration.pm +++ b/lib/App/Netdisco/Configuration.pm @@ -84,6 +84,7 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') }) config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ]; } # fix up device_auth (or create it from old snmp_auth and community settings) +# also imports legacy sshcollcetor config config->{'device_auth'} = [ App::Netdisco::Util::DeviceAuth::fixup_device_auth() ]; diff --git a/lib/App/Netdisco/Transport/CLI.pm b/lib/App/Netdisco/Transport/CLI.pm new file mode 100644 index 00000000..6f865c7b --- /dev/null +++ b/lib/App/Netdisco/Transport/CLI.pm @@ -0,0 +1,115 @@ +package App::Netdisco::Transport::CLI; + +use Dancer qw/:syntax :script/; + +use App::Netdisco::Util::Device 'get_device'; +use Module::Load (); +use Net::OpenSSH; +use Try::Tiny; + +use base 'Dancer::Object::Singleton'; + +=head1 NAME + +App::Netdisco::Transport::CLI + +=head1 DESCRIPTION + +Returns an object which has an active SSH connection which can be used +for some actions such as arpnip. + + my $cli = App::Netdisco::Transport::CLI->session_for( ... ); + +=cut + +__PACKAGE__->attributes(qw/ sessions /); + +sub init { + my ( $class, $self ) = @_; + $self->sessions( {} ); + return $self; +} + +=head1 session_for( $ip ) + +Given an IP address, returns an object instance configured for and connected +to that device. + +Returns C if the connection fails. + +=cut + +{ + package MySession; + use Moo; + + has 'ssh' => ( is => 'rw' ); + has 'auth' => ( is => 'rw' ); + has 'host' => ( is => 'rw' ); + has 'platform' => ( is => 'rw' ); + + sub arpnip { + my $self = shift; + $self->platform->arpnip(@_, $self->host, $self->ssh, $self->auth); + } +} + +sub session_for { + my ($class, $ip) = @_; + + my $device = get_device($ip) or return undef; + my $sessions = $class->instance->sessions or return undef; + + return $sessions->{$device->ip} if exists $sessions->{$device->ip}; + debug sprintf 'cli session cache warm: [%s]', $device->ip; + + my $auth = (setting('device_auth') || []); + if (1 != scalar @$auth) { + error sprintf " [%s] require only one matching auth stanza", $device->ip; + return undef; + } + $auth = $auth->[0]; + + my @master_opts = qw(-o BatchMode=no); + push(@master_opts, @{$auth->{ssh_master_opts}}) + if $auth->{ssh_master_opts}; + + $Net::OpenSSH::debug = $ENV{SSH_TRACE}; + my $ssh = Net::OpenSSH->new( + $device->ip, + user => $auth->{username}, + password => $auth->{password}, + timeout => 30, + async => 0, + default_stderr_file => '/dev/null', + master_opts => \@master_opts + ); + + if ($ssh->error) { + error sprintf " [%s] ssh connection error [%s]", $device->ip, $ssh->error; + return undef; + } + elsif (! $ssh) { + error sprintf " [%s] Net::OpenSSH instantiation error", $device->ip; + return undef; + } + + my $platform = "App::Netdisco::SSHCollector::Platform::" . $auth->{platform}; + my $happy = false; + try { + Module::Load::load $platform; + $happy = true; + } catch { error $_ }; + return unless $happy; + + my $sess = MySession->new( + ssh => $ssh, + auth => $auth, + host => $device->ip, + platform => $platform->new(), + ); + + return ($sessions->{$device->ip} = $sess); +} + +true; diff --git a/lib/App/Netdisco/Transport/SNMP.pm b/lib/App/Netdisco/Transport/SNMP.pm index 36cf57b3..f342155b 100644 --- a/lib/App/Netdisco/Transport/SNMP.pm +++ b/lib/App/Netdisco/Transport/SNMP.pm @@ -25,7 +25,7 @@ App::Netdisco::Transport::SNMP Singleton for SNMP connections. Returns cached L instance for a given device IP, or else undef. All methods are class methods, for example: - App::Netdisco::Transport::SNMP->reader_for( ... ); + my $snmp = App::Netdisco::Transport::SNMP->reader_for( ... ); =cut diff --git a/lib/App/Netdisco/Util/DeviceAuth.pm b/lib/App/Netdisco/Util/DeviceAuth.pm index bb7f79f5..31f68472 100644 --- a/lib/App/Netdisco/Util/DeviceAuth.pm +++ b/lib/App/Netdisco/Util/DeviceAuth.pm @@ -63,7 +63,23 @@ sub fixup_device_auth { die "error: config: stanza in device_auth must have a tag\n" if not $stanza->{tag} and exists $stanza->{user}; - push @new_stanzas, $stanza + push @new_stanzas, $stanza; + } + + # import legacy sshcollector configuration + my $sshcollector = (setting('sshcollector') || []); + foreach my $stanza (@$sshcollector) { + # defaults + $stanza->{driver} = 'cli'; + $stanza->{read} = 1; + $stanza->{no} ||= []; + + # fixups + $stanza->{only} ||= [ scalar delete $stanza->{ip} || + scalar delete $stanza->{hostname} ]; + $stanza->{username} = scalar delete $stanza->{user}; + + push @new_stanzas, $stanza; } # legacy config diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm index 6805737a..3ac3c307 100644 --- a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm @@ -3,7 +3,7 @@ package App::Netdisco::Worker::Plugin::Arpnip::Nodes; use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; - +use App::Netdisco::Transport::CLI (); use App::Netdisco::Transport::SNMP (); use App::Netdisco::Util::Node qw/check_mac store_arp/; use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; @@ -18,9 +18,9 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { or return Status->defer("arpnip failed: could not SNMP connect to $device"); # get v4 arp table - my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); + my $v4 = get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr); # get v6 neighbor cache - my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); + my $v6 = get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); # would be possible just to use now() on updated records, but by using this # same value for them all, we _can_ if we want add a job at the end to @@ -41,7 +41,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { }); # get an arp table (v4 or v6) -sub get_arps { +sub get_arps_snmp { my ($device, $paddr, $netaddr) = @_; my @arps = (); @@ -63,4 +63,45 @@ sub get_arps { return $resolved_ips; } +register_worker({ phase => 'main', driver => 'cli' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + my $cli = App::Netdisco::Transport::CLI->session_for($device) + or return Status->defer("arpnip failed: could not SSH connect to $device"); + + # should be both v4 and v6 + my $arps = get_arps_cli($device, [$cli->arpnip]); + + # update node_ip with ARP and Neighbor Cache entries + my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; + store_arp(\%$_, $now) for @$arps; + debug sprintf ' [%s] arpnip - processed %s ARP / IPv6 Neighbor Cache entries', + $device->ip, scalar @$arps; + + $device->update({last_arpnip => \$now}); + return Status->done("Ended arpnip for $device"); +}); + +sub get_arps_cli { + my ($device, $entries) = @_; + my @arps = (); + $entries ||= []; + + foreach my $entry (@$entries) { + next unless check_mac($entry->{mac}, $device); + push @arps, { + node => $entry->{mac}, + ip => $entry->{ip}, + dns => $entry->{dns}, + }; + } + + debug sprintf ' resolving %d ARP entries with max %d outstanding requests', + scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; + my $resolved_ips = hostnames_resolve_async(\@arps); + + return $resolved_ips; +} + true; diff --git a/share/config.yml b/share/config.yml index 9767019a..5422e4ea 100644 --- a/share/config.yml +++ b/share/config.yml @@ -216,6 +216,7 @@ device_identity: [] community: [] community_rw: [] device_auth: [] +use_legacy_sshcollector: false get_credentials: "" bulkwalk_off: false bulkwalk_no: [] diff --git a/xt/00-compile.t b/xt/00-compile.t index 307058af..f9e3b606 100644 --- a/xt/00-compile.t +++ b/xt/00-compile.t @@ -21,8 +21,8 @@ use Test::Compile; my $test = Test::Compile->new(); -my @plfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pl_files(); -my @pmfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pm_files(); +my @plfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pl_files(); +my @pmfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pm_files(); $test->ok($test->pl_file_compiles($_), "$_ compiles") for @plfiles; $test->ok($test->pm_file_compiles($_), "$_ compiles") for @pmfiles;