Integrate netdisco-sshcollector into Worker::Plugin architecture (#489)

* Initial integration of sshcollector into Worker::Plugin architecture

 * NOT FULLY FUNCTIONAL - this is only to discuss some issues for now
 * add NodesBySSH.pm
 * update Build.PL and config.yml to integrate the new module

* Further integration of sshcollector into Worker::Plugin architecture

 * NOT FULLY FUNCTIONAL - this is only to discuss some issues for now
 * added App::Netdisco::Transport::CLI loosely based on ::SNMP counterpart
 * switched to the more prevalent two-space tabs style
 * removed various TBD items, some new ones

* Further steps to integration of sshcollector into Worker::Plugin architecture

 * cleaned up code
 * added various error handling
 * warning for bin/netdisco-sshcollector deprecation
 * device_auth allows passing master_opts to Net::OpenSSH
 * netdisco-do -D also toggles Net::OpenSSH debug

* Merged NodesBySSH.pm into Nodes.pm

 * see https://github.com/netdisco/netdisco/pull/489#pullrequestreview-205603516

* Further integration of sshcollector into Worker::Plugin architecture

 * add snmp_arpnip_also option to sshcollector device_auth
 * cleanup code

* Remove big TBD: comment from CLI.pm as doc is updated now

Build.PL

@@ -37,6 +37,7 @@ Module::Build->new(
     'Dancer::Plugin::Auth::Extensible' => '0.30',
     'Dancer::Plugin::Passphrase' => '2.0.1',
     'Dancer::Session::Cookie' => '0.27',
+    'Expect' => '0',
     'File::ShareDir' => '1.03',
     'File::Slurper' => '0.009',
     'Guard' => '1.022',
@@ -54,6 +55,7 @@ Module::Build->new(
     'Net::Domain' => '1.23',
     'Net::DNS' => '0.72',
     'Net::LDAP' => '0',
+    'Net::OpenSSH' => '0',
     'NetAddr::MAC' => '0.93',
     'NetAddr::IP' => '4.068',
     'Opcode' => '1.07',
@@ -90,8 +92,6 @@ Module::Build->new(
   recommends => {
     'Graph' => '0',
     'GraphViz' => '0',
-    'Net::OpenSSH' => '0',
-    'Expect' => '0',
   },
   test_requires => {
     'Test::More' => '1.302083',

bin/netdisco-sshcollector

@@ -78,13 +78,17 @@ if ($opensshdebug){
     $Net::OpenSSH::debug = ~0;
 }
 
+
 MCE::Loop::init { chunk_size => 1, max_workers => $workers };
 my %stats;
 $stats{entry} = 0;
 
+deprecation_warning();
 exit main();
 
 sub main {
+
+
     my @input = @{ setting('sshcollector') };
 
     if ($device){
@@ -176,6 +180,15 @@ sub store_arpentries {
     }
 }
 
+sub deprecation_warning {
+    print "\n";
+    warning sprintf "DEPRECATION WARNING\n" .
+                    "This script and the sshcollector setting will be removed in a future release!\n".
+                    "See this document to migrate to the new sshcollector integrated into\n" .
+                    "regular netdisco-do/netdisco-daemon arpnip:\n" .
+                    "https://github.com/netdisco/netdisco/wiki/bin-sshcollector-deprecation\n\n";
+}
+
 =head1 NAME
 
 netdisco-sshcollector - Collect ARP data for Netdisco from devices without

lib/App/Netdisco/Transport/CLI.pm

@@ -0,0 +1,84 @@
+package App::Netdisco::Transport::CLI;
+
+use Dancer qw/:syntax :script/;
+use Dancer::Plugin::DBIC 'schema';
+
+use App::Netdisco::Util::Device 'get_device';
+use App::Netdisco::Util::Permission ':all';
+use Module::Load ();
+use Path::Class 'dir';
+use NetAddr::IP::Lite ':lower';
+use List::Util qw/pairkeys pairfirst/;
+
+use base 'Dancer::Object::Singleton';
+
+=head1 NAME
+
+App::Netdisco::Transport::CLI
+
+=head1 DESCRIPTION
+
+Singleton for CLI connections modelled after L<App::Netdisco::Transport::SNMP> but currently 
+with minimal functionality. Returns a L<Net::OpenSSH> instance for a given device IP. Limited 
+to device_auth stanzas tagged sshcollector. Always returns a new connection which the caller 
+is supposed to close (or it will be closed when going out of scope)
+
+=cut
+
+
+sub init {
+  my ( $class, $self ) = @_;
+  return $self;
+}
+
+=head1 session_for( $ip, $tag )
+
+Given an IP address and a tag, returns an L<Net::OpenSSH> instance configured for and
+connected to that device, as well as the C<device_auth> entry that was chosen for the device.  
+
+Returns C<undef> if the connection fails.
+
+=cut
+
+sub session_for {
+  my ($class, $ip, $tag) = @_;
+  my $device = get_device($ip) or return undef;
+
+  my $device_auth = [grep { $_->{tag} eq $tag } @{setting('device_auth')}];
+
+  # Currently just the first match is used. Warn if there are more.
+  my $selected_auth = $device_auth->[0];
+
+  if (@{$device_auth} > 1){
+    warning sprintf " [%s] Transport::CLI - found %d matching entries in device_auth, using the first one", 
+      $device->ip, scalar @{$device_auth};
+  }
+
+  my @master_opts = qw(-o BatchMode=no);
+  push(@master_opts, @{$selected_auth->{ssh_master_opts}}) if $selected_auth->{ssh_master_opts};
+
+  my $ssh = Net::OpenSSH->new(
+    $device->ip,
+    user => $selected_auth->{username},
+    password => $selected_auth->{password},
+    timeout => 30,
+    async => 0,
+    default_stderr_file => '/dev/null',
+    master_opts => \@master_opts
+  );
+
+  my $CONFIG = config();
+  $Net::OpenSSH::debug = ~0 if $CONFIG->{log} eq 'debug';
+
+  if ($ssh->error){
+    error sprintf " [%s] Transport::CLI - ssh connection error [%s]", $device->ip, $ssh->error;
+    return undef;
+  }elsif (!$ssh){
+    error sprintf " [%s] Transport::CLI - Net::OpenSSH instantiation error", $device->ip;
+    return undef;
+  }else{
+    return ($ssh, $selected_auth);
+  }
+}
+
+true;

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -3,24 +3,27 @@ package App::Netdisco::Worker::Plugin::Arpnip::Nodes;
 use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
-
+use App::Netdisco::Transport::CLI ();
 use App::Netdisco::Transport::SNMP ();
 use App::Netdisco::Util::Node qw/check_mac store_arp/;
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
 use Dancer::Plugin::DBIC 'schema';
 use Time::HiRes 'gettimeofday';
+use Module::Load ();
+use Net::OpenSSH;
+use Try::Tiny;
 
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my ($job, $workerconf) = @_;
 
   my $device = $job->device;
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
-    or return Status->defer("arpnip failed: could not SNMP connect to $device");
+    or return Status->defer("arpnip snmp failed: could not SNMP connect to $device");
 
   # get v4 arp table
-  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
+  my $v4 = get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr);
   # get v6 neighbor cache
-  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
+  my $v6 = get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
 
   # would be possible just to use now() on updated records, but by using this
   # same value for them all, we _can_ if we want add a job at the end to
@@ -29,19 +32,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
 
   # update node_ip with ARP and Neighbor Cache entries
   store_arp(\%$_, $now) for @$v4;
-  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
+  debug sprintf ' [%s] arpnip snmp - processed %s ARP Cache entries',
     $device->ip, scalar @$v4;
 
   store_arp(\%$_, $now) for @$v6;
-  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
+  debug sprintf ' [%s] arpnip snmp - processed %s IPv6 Neighbor Cache entries',
     $device->ip, scalar @$v6;
 
   $device->update({last_arpnip => \$now});
-  return Status->done("Ended arpnip for $device");
+  return Status->done("Ended arpnip snmp for $device");
 });
 
 # get an arp table (v4 or v6)
-sub get_arps {
+sub get_arps_snmp {
   my ($device, $paddr, $netaddr) = @_;
   my @arps = ();
 
@@ -63,4 +66,78 @@ sub get_arps {
   return $resolved_ips;
 }
 
+register_worker({ phase => 'main', driver => 'cli' }, sub {
+    my ($job, $workerconf) = @_;
+
+    my $device = $job->device;
+    my ($ssh, $selected_auth) = App::Netdisco::Transport::CLI->session_for($device->ip, "sshcollector");
+
+    if (get_arps_cli($device, $ssh, $selected_auth)){
+      my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+      $device->update({last_arpnip => \$now});
+      my $endmsg = "Ended arpnip cli for $device"; 
+
+      if ($selected_auth->{'snmp_arpnip_also'}){
+        $endmsg .= ", now running arpnip due to snmp_arpnip_also"; 
+        info sprintf " [%s] arpnip cli - $endmsg", $device->ip;
+        return Status->info($endmsg);
+      }else{
+        info sprintf " [%s] arpnip cli - $endmsg", $device->ip;
+        return Status->done($endmsg);
+      }
+    }else{
+      Status->defer("arpnip cli failed");
+    }
+
+});
+
+sub get_arps_cli {
+  my ($device, $ssh, $selected_auth) = @_;
+
+
+  unless ($ssh){
+    my $msg = "could not connect to $device with SSH, deferring job"; 
+    warning sprintf " [%s] arpnip cli - %s", $device->ip, $msg;
+    return undef;
+  }
+
+  my $class = "App::Netdisco::SSHCollector::Platform::".$selected_auth->{platform};
+  debug sprintf " [%s] arpnip cli - delegating to platform module %s", $device->ip, $class;
+
+  my $load_failed = 0;
+  try {
+    Module::Load::load $class;
+  } catch {
+    warning sprintf " [%s] arpnip cli - failed to load %s: %s", $device->ip, $class, substr($_, 0, 50)."...";
+    $load_failed = 1;
+  };
+  return undef if $load_failed;
+
+  my $platform_class = $class->new();
+  my $arpentries = [ $platform_class->arpnip($device->ip, $ssh, $selected_auth) ];
+
+  if (not scalar @$arpentries) {
+    warning sprintf " [%s] WARNING: no entries received from device", $device->ip;
+  }
+
+  hostnames_resolve_async($arpentries);
+
+  foreach my $arpentry ( @$arpentries ) {
+
+    # skip broadcast/vrrp/hsrp and other weirdos
+    next unless check_mac( $arpentry->{mac} );
+
+    debug sprintf ' [%s] arpnip cli - stored entry: %s / %s / %s',
+    $device->ip, $arpentry->{mac}, $arpentry->{ip}, 
+    $arpentry->{dns} if defined $arpentry->{dns};
+    store_arp({
+        node => $arpentry->{mac},
+        ip => $arpentry->{ip},
+        dns => $arpentry->{dns},
+      });
+  }
+
+  return 1;
+}
+
 true;