Merge branch 'master' into og-api-tokens-simple

lib/App/Netdisco/Backend/Job.pm

@@ -93,7 +93,7 @@ phase.
 
 sub finalise_status {
   my $job = shift;
-  # use DDP; p $job->_statuslist;
+  # use DDP; p $job->_statuslist;
 
   # fallback
   $job->status('error');
@@ -103,7 +103,7 @@ sub finalise_status {
 
   foreach my $status (reverse @{ $job->_statuslist }) {
     next if $status->phase 
-      and $status->phase !~ m/^(?:check|early|main)$/;
+      and $status->phase !~ m/^(?:check|early|main|store|late)$/;
 
     # done() from check phase should not be the action's done()
     next if $status->phase eq 'check' and $status->is_ok;

lib/App/Netdisco/Configuration.pm

@@ -85,6 +85,7 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') })
   config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ];
 }
 # fix up device_auth (or create it from old snmp_auth and community settings)
+# also imports legacy sshcollcetor config
 config->{'device_auth'}
   = [ App::Netdisco::Util::DeviceAuth::fixup_device_auth() ];
 

lib/App/Netdisco/Transport/SNMP.pm

@@ -25,7 +25,7 @@ App::Netdisco::Transport::SNMP
 Singleton for SNMP connections. Returns cached L<SNMP::Info> instance for a
 given device IP, or else undef. All methods are class methods, for example:
 
- App::Netdisco::Transport::SNMP->reader_for( ... );
+ my $snmp = App::Netdisco::Transport::SNMP->reader_for( ... );
 
 =cut
 

lib/App/Netdisco/Transport/SSH.pm

@@ -0,0 +1,115 @@
+package App::Netdisco::Transport::SSH;
+
+use Dancer qw/:syntax :script/;
+
+use App::Netdisco::Util::Device 'get_device';
+use Module::Load ();
+use Net::OpenSSH;
+use Try::Tiny;
+
+use base 'Dancer::Object::Singleton';
+
+=head1 NAME
+
+App::Netdisco::Transport::SSH
+
+=head1 DESCRIPTION
+
+Returns an object which has an active SSH connection which can be used
+for some actions such as arpnip.
+
+ my $cli = App::Netdisco::Transport::SSH->session_for( ... );
+
+=cut
+
+__PACKAGE__->attributes(qw/ sessions /);
+
+sub init {
+  my ( $class, $self ) = @_;
+  $self->sessions( {} );
+  return $self;
+}
+
+=head1 session_for( $ip )
+
+Given an IP address, returns an object instance configured for and connected
+to that device.
+
+Returns C<undef> if the connection fails.
+
+=cut
+
+{
+  package MySession;
+  use Moo;
+
+  has 'ssh'  => ( is => 'rw' );
+  has 'auth' => ( is => 'rw' );
+  has 'host' => ( is => 'rw' );
+  has 'platform' => ( is => 'rw' );
+
+  sub arpnip {
+    my $self = shift;
+    $self->platform->arpnip(@_, $self->host, $self->ssh, $self->auth);
+  }
+}
+
+sub session_for {
+  my ($class, $ip) = @_;
+
+  my $device = get_device($ip) or return undef;
+  my $sessions = $class->instance->sessions or return undef;
+
+  return $sessions->{$device->ip} if exists $sessions->{$device->ip};
+  debug sprintf 'cli session cache warm: [%s]', $device->ip;
+
+  my $auth = (setting('device_auth') || []);
+  if (1 != scalar @$auth) {
+    error sprintf " [%s] require only one matching auth stanza", $device->ip;
+    return undef;
+  }
+  $auth = $auth->[0];
+
+  my @master_opts = qw(-o BatchMode=no);
+  push(@master_opts, @{$auth->{ssh_master_opts}})
+    if $auth->{ssh_master_opts};
+
+  $Net::OpenSSH::debug = $ENV{SSH_TRACE};
+  my $ssh = Net::OpenSSH->new(
+    $device->ip,
+    user => $auth->{username},
+    password => $auth->{password},
+    timeout => 30,
+    async => 0,
+    default_stderr_file => '/dev/null',
+    master_opts => \@master_opts
+  );
+
+  if ($ssh->error) {
+    error sprintf " [%s] ssh connection error [%s]", $device->ip, $ssh->error;
+    return undef;
+  }
+  elsif (! $ssh) {
+    error sprintf " [%s] Net::OpenSSH instantiation error", $device->ip;
+    return undef;
+  }
+
+  my $platform = "App::Netdisco::SSHCollector::Platform::" . $auth->{platform};
+  my $happy = false;
+  try {
+    Module::Load::load $platform;
+    $happy = true;
+  } catch { error $_ };
+  return unless $happy;
+
+  my $sess = MySession->new(
+    ssh  => $ssh,
+    auth => $auth,
+    host => $device->ip,
+    platform => $platform->new(),
+  );
+
+  return ($sessions->{$device->ip} = $sess);
+}
+
+true;

lib/App/Netdisco/Util/DeviceAuth.pm

@@ -63,7 +63,23 @@ sub fixup_device_auth {
     die "error: config: stanza in device_auth must have a tag\n"
       if not $stanza->{tag} and exists $stanza->{user};
 
-    push @new_stanzas, $stanza
+    push @new_stanzas, $stanza;
+  }
+
+  # import legacy sshcollector configuration
+  my $sshcollector = (setting('sshcollector') || []);
+  foreach my $stanza (@$sshcollector) {
+    # defaults
+    $stanza->{driver} = 'cli';
+    $stanza->{read} = 1;
+    $stanza->{no}   ||= [];
+
+    # fixups
+    $stanza->{only} ||= [ scalar delete $stanza->{ip} ||
+                          scalar delete $stanza->{hostname} ];
+    $stanza->{username} = scalar delete $stanza->{user};
+
+    push @new_stanzas, $stanza;
   }
 
   # legacy config 

lib/App/Netdisco/Util/Port.pm

@@ -219,7 +219,7 @@ Returns true if the C<$port> L<DBIx::Class> object has a phone connected.
 =cut
 
 sub port_has_phone {
-  return (shift)->with_properties->remote_is_phone;
+  return (shift)->properties->remote_is_phone;
 }
 
 1;

lib/App/Netdisco/Util/SNMP.pm

@@ -100,11 +100,19 @@ sub snmp_comm_reindex {
       }
       $prefix ||= 'vlan-';
 
-      debug
-        sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
+      if ($vlan =~ /^[0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
         $device->ip, $prefix, $vlan, $ver, $snmp->class;
-      $vlan ? $snmp->update(Context => ($prefix . $vlan))
-            : $snmp->update(Context => '');
+        $snmp->update(Context => ($prefix . $vlan));
+      } elsif ($vlan =~ /^[a-z0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s" (ver: %s, class: %s)',
+          $device->ip, $vlan, $ver, $snmp->class;
+        $snmp->update(Context => ($vlan));
+      } else { 
+        debug sprintf '[%s] reindexing without context (ver: %s, class: %s)',
+          $device->ip, $ver, $snmp->class;
+        $snmp->update(Context => ''); 
+      }
   }
   else {
       my $comm = $snmp->snmp_comm;

lib/App/Netdisco/Worker/Loader.pm

@@ -12,7 +12,9 @@ use namespace::clean;
 has [qw/workers_check
         workers_early
         workers_main
-        workers_user/] => ( is => 'rw' );
+        workers_user
+        workers_store
+        workers_late/] => ( is => 'rw' );
 
 sub load_workers {
   my $self = shift;
@@ -37,7 +39,7 @@ sub load_workers {
   my $workers = vars->{'workers'}->{$action} || {};
   #use DDP; p vars->{'workers'};
 
-  foreach my $phase (qw/check early main user/) {
+  foreach my $phase (qw/check early main user store late/) {
     my $pname = "workers_${phase}";
     my @wset = ();
 

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -1,15 +1,43 @@
 package App::Netdisco::Worker::Plugin::Arpnip::Nodes;
 
 use Dancer ':syntax';
+use Dancer::Plugin::DBIC 'schema';
+
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
+use App::Netdisco::Transport::SSH ();
 use App::Netdisco::Transport::SNMP ();
+
 use App::Netdisco::Util::Node qw/check_mac store_arp/;
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
-use Dancer::Plugin::DBIC 'schema';
+
+use NetAddr::IP::Lite ':lower';
 use Time::HiRes 'gettimeofday';
 
+register_worker({ phase => 'store' }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  # would be possible just to use now() on updated records, but by using this
+  # same value for them all, we _can_ if we want add a job at the end to
+  # select and do something with the updated set (no reason to yet, though)
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  # update node_ip with ARP and Neighbor Cache entries
+
+  store_arp(\%$_, $now) for @{ vars->{'v4arps'} };
+  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
+    $device->ip, scalar @{ vars->{'v4arps'} };
+
+  store_arp(\%$_, $now) for @{ vars->{'v6arps'} };
+  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
+    $device->ip, scalar @{ vars->{'v6arps'} };
+
+  $device->update({last_arpnip => \$now});
+  return Status->done("Ended arpnip for $device");
+});
+
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my ($job, $workerconf) = @_;
 
@@ -17,31 +45,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
     or return Status->defer("arpnip failed: could not SNMP connect to $device");
 
-  # get v4 arp table
-  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
-  # get v6 neighbor cache
-  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
+  # cache v4 arp table
+  push @{ vars->{'v4arps'} },
+    @{ get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr) };
 
-  # would be possible just to use now() on updated records, but by using this
-  # same value for them all, we _can_ if we want add a job at the end to
-  # select and do something with the updated set (no reason to yet, though)
-  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+  # cache v6 neighbor cache
+  push @{ vars->{'v6arps'} },
+    @{get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr) };
 
-  # update node_ip with ARP and Neighbor Cache entries
-  store_arp(\%$_, $now) for @$v4;
-  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
-    $device->ip, scalar @$v4;
-
-  store_arp(\%$_, $now) for @$v6;
-  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
-    $device->ip, scalar @$v6;
-
-  $device->update({last_arpnip => \$now});
-  return Status->done("Ended arpnip for $device");
+  return Status->info("Gathered arp caches from $device");
 });
 
 # get an arp table (v4 or v6)
-sub get_arps {
+sub get_arps_snmp {
   my ($device, $paddr, $netaddr) = @_;
   my @arps = ();
 
@@ -63,4 +79,46 @@ sub get_arps {
   return $resolved_ips;
 }
 
+register_worker({ phase => 'main', driver => 'cli' }, sub {
+    my ($job, $workerconf) = @_;
+
+    my $device = $job->device;
+    my $cli = App::Netdisco::Transport::SSH->session_for($device)
+      or return Status->defer("arpnip failed: could not SSH connect to $device");
+
+    # should be both v4 and v6
+    my @arps = @{ get_arps_cli($device, [$cli->arpnip]) };
+
+    # cache v4 arp table
+    push @{ vars->{'v4arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits ==  32 } @arps;
+
+    # cache v6 neighbor cache
+    push @{ vars->{'v6arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits == 128 } @arps;
+
+    return Status->info("Gathered arp caches from $device");
+});
+
+sub get_arps_cli {
+  my ($device, $entries) = @_;
+  my @arps = ();
+  $entries ||= [];
+
+  foreach my $entry (@$entries) {
+    next unless check_mac($entry->{mac}, $device);
+    push @arps, {
+        node => $entry->{mac},
+        ip   => $entry->{ip},
+        dns  => $entry->{dns},
+    };
+  }
+
+  debug sprintf ' resolving %d ARP entries with max %d outstanding requests',
+    scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_ips = hostnames_resolve_async(\@arps);
+
+  return $resolved_ips;
+}
+
 true;

lib/App/Netdisco/Worker/Runner.pm

@@ -69,7 +69,7 @@ sub run {
 
     # run other phases
     if ($job->check_passed) {
-      $self->run_workers("workers_${_}") for qw/early main user/;
+      $self->run_workers("workers_${_}") for qw/early main user store late/;
     }
   };