Merge branch 'master' into og-api-tokens-simple

Build.PL

@@ -38,6 +38,7 @@ Module::Build->new(
     'Dancer::Plugin::Passphrase' => '2.0.1',
     'Dancer::Plugin::Swagger' => '0',
     'Dancer::Session::Cookie' => '0.27',
+    'Expect' => '0',
     'File::ShareDir' => '1.03',
     'File::Slurper' => '0.009',
     'Guard' => '1.022',
@@ -55,6 +56,7 @@ Module::Build->new(
     'Net::Domain' => '1.23',
     'Net::DNS' => '0.72',
     'Net::LDAP' => '0',
+    'Net::OpenSSH' => '0',
     'NetAddr::MAC' => '0.93',
     'NetAddr::IP' => '4.068',
     'Opcode' => '1.07',
@@ -91,8 +93,6 @@ Module::Build->new(
   recommends => {
     'Graph' => '0',
     'GraphViz' => '0',
-    'Net::OpenSSH' => '0',
-    'Expect' => '0',
   },
   test_requires => {
     'Test::More' => '1.302083',

Changes

@@ -1,3 +1,25 @@
+2.041001 - 2019-03-15
+
+  [ENHANCEMENTS]
+
+  * add "store" and "late" phases to backend workers
+  * documentation updates
+
+  [BUG FIXES]
+
+  * #539 fix an issue with VLAN reindixing for VRFs (earendilfr)
+  * fix device port change check
+
+2.041000 - 2019-03-12
+
+  [NEW FEATURES]
+
+  * netdisco-sshcollector no longer required - the functionality (arpnip via
+    ssh) will be run within netdisco's core schedule (rc9000 and ollyg)
+
+  * get_credentials replaces get_community and accepts any device_auth stanza
+    in JSON format (ollyg)
+
 2.040007 - 2019-03-06
 
   [BUG FIXES]

MANIFEST

@@ -110,7 +110,9 @@ lib/App/Netdisco/SSHCollector/Platform/Linux.pm
 lib/App/Netdisco/SSHCollector/Platform/NXOS.pm
 lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
 lib/App/Netdisco/Transport/SNMP.pm
+lib/App/Netdisco/Transport/SSH.pm
 lib/App/Netdisco/Util/Device.pm
+lib/App/Netdisco/Util/DeviceAuth.pm
 lib/App/Netdisco/Util/DNS.pm
 lib/App/Netdisco/Util/ExpandParams.pm
 lib/App/Netdisco/Util/FastResolver.pm
@@ -485,6 +487,7 @@ share/views/sidebar/report/subnets.tt
 share/views/sidebar/search/device.tt
 share/views/sidebar/search/node.tt
 share/views/sidebar/search/port.tt
+xt/00-compile.t
 xt/10-sort_port.t
 xt/11-portsort.t
 xt/20-checkacl.t

META.json

@@ -29,10 +29,8 @@
       },
       "runtime" : {
          "recommends" : {
-            "Expect" : "0",
             "Graph" : "0",
-            "GraphViz" : "0",
+            "GraphViz" : "0"
-            "Net::OpenSSH" : "0"
          },
          "requires" : {
             "Algorithm::Cron" : "0.07",
@@ -52,6 +50,7 @@
             "Dancer::Plugin::Passphrase" : "v2.0.1",
             "Dancer::Session::Cookie" : "0.27",
             "Data::Printer" : "0",
+            "Expect" : "0",
             "File::ShareDir" : "1.03",
             "File::Slurper" : "0.009",
             "Guard" : "1.022",
@@ -69,6 +68,7 @@
             "Net::DNS" : "0.72",
             "Net::Domain" : "1.23",
             "Net::LDAP" : "0",
+            "Net::OpenSSH" : "0",
             "NetAddr::IP" : "4.068",
             "NetAddr::MAC" : "0.93",
             "Opcode" : "1.07",
@@ -110,6 +110,7 @@
       "test" : {
          "requires" : {
             "Env::Path" : "0",
+            "Test::Compile" : "0",
             "Test::File::ShareDir::Dist" : "0",
             "Test::More" : "1.302083"
          }
@@ -118,7 +119,7 @@
    "provides" : {
       "App::Netdisco" : {
          "file" : "lib/App/Netdisco.pm",
-         "version" : "2.040007"
+         "version" : "2.041001"
       },
       "App::Netdisco::AnyEvent::Nbtstat" : {
          "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm"
@@ -409,12 +410,18 @@
       "App::Netdisco::Transport::SNMP" : {
          "file" : "lib/App/Netdisco/Transport/SNMP.pm"
       },
+      "App::Netdisco::Transport::SSH" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
+      },
       "App::Netdisco::Util::DNS" : {
          "file" : "lib/App/Netdisco/Util/DNS.pm"
       },
       "App::Netdisco::Util::Device" : {
          "file" : "lib/App/Netdisco/Util/Device.pm"
       },
+      "App::Netdisco::Util::DeviceAuth" : {
+         "file" : "lib/App/Netdisco/Util/DeviceAuth.pm"
+      },
       "App::Netdisco::Util::ExpandParams" : {
          "file" : "lib/App/Netdisco/Util/ExpandParams.pm"
       },
@@ -783,6 +790,9 @@
       },
       "Dancer::Template::NetdiscoTemplateToolkit" : {
          "file" : "lib/Dancer/Template/NetdiscoTemplateToolkit.pm"
+      },
+      "MySession" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
       }
    },
    "release_status" : "stable",
@@ -792,7 +802,7 @@
       },
       "homepage" : "http://netdisco.org/",
       "license" : [
-         "http://opensource.org/licenses/bsd-license.php"
+         "http://opensource.org/licenses/BSD-3-Clause"
       ],
       "repository" : {
          "url" : "https://github.com/netdisco/netdisco"
@@ -800,6 +810,6 @@
       "x_IRC" : "irc://irc.freenode.org/#netdisco",
       "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users"
    },
-   "version" : "2.040007",
+   "version" : "2.041001",
    "x_serialization_backend" : "JSON::PP version 2.97001"
 }

META.yml

@@ -7,6 +7,7 @@ build_requires:
   ExtUtils::Config: '0'
   ExtUtils::Helpers: '0'
   ExtUtils::InstallPaths: '0'
+  Test::Compile: '0'
   Test::File::ShareDir::Dist: '0'
   Test::More: '1.302083'
 configure_requires:
@@ -22,7 +23,7 @@ name: App-Netdisco
 provides:
   App::Netdisco:
     file: lib/App/Netdisco.pm
-    version: '2.040007'
+    version: '2.041001'
   App::Netdisco::AnyEvent::Nbtstat:
     file: lib/App/Netdisco/AnyEvent/Nbtstat.pm
   App::Netdisco::Backend::Job:
@@ -216,10 +217,14 @@ provides:
     file: lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
   App::Netdisco::Transport::SNMP:
     file: lib/App/Netdisco/Transport/SNMP.pm
+  App::Netdisco::Transport::SSH:
+    file: lib/App/Netdisco/Transport/SSH.pm
   App::Netdisco::Util::DNS:
     file: lib/App/Netdisco/Util/DNS.pm
   App::Netdisco::Util::Device:
     file: lib/App/Netdisco/Util/Device.pm
+  App::Netdisco::Util::DeviceAuth:
+    file: lib/App/Netdisco/Util/DeviceAuth.pm
   App::Netdisco::Util::ExpandParams:
     file: lib/App/Netdisco/Util/ExpandParams.pm
   App::Netdisco::Util::FastResolver:
@@ -466,11 +471,11 @@ provides:
     file: lib/App/Netdisco/Worker/Status.pm
   Dancer::Template::NetdiscoTemplateToolkit:
     file: lib/Dancer/Template/NetdiscoTemplateToolkit.pm
+  MySession:
+    file: lib/App/Netdisco/Transport/SSH.pm
 recommends:
-  Expect: '0'
   Graph: '0'
   GraphViz: '0'
-  Net::OpenSSH: '0'
 requires:
   Algorithm::Cron: '0.07'
   AnyEvent: '7.05'
@@ -489,6 +494,7 @@ requires:
   Dancer::Plugin::Passphrase: v2.0.1
   Dancer::Session::Cookie: '0.27'
   Data::Printer: '0'
+  Expect: '0'
   File::ShareDir: '1.03'
   File::Slurper: '0.009'
   Guard: '1.022'
@@ -506,6 +512,7 @@ requires:
   Net::DNS: '0.72'
   Net::Domain: '1.23'
   Net::LDAP: '0'
+  Net::OpenSSH: '0'
   NetAddr::IP: '4.068'
   NetAddr::MAC: '0.93'
   Opcode: '1.07'
@@ -547,7 +554,7 @@ resources:
   MailingList: https://lists.sourceforge.net/lists/listinfo/netdisco-users
   bugtracker: https://github.com/netdisco/netdisco/issues
   homepage: http://netdisco.org/
-  license: http://opensource.org/licenses/bsd-license.php
+  license: http://opensource.org/licenses/BSD-3-Clause
   repository: https://github.com/netdisco/netdisco
-version: '2.040007'
+version: '2.041001'
 x_serialization_backend: 'CPAN::Meta::YAML version 0.018'

bin/netdisco-sshcollector

@@ -74,6 +74,9 @@ $ENV{DBIC_TRACE} ||= $sqltrace;
 # reconfigure logging to force console output
 Dancer::Logger->init('console', $CONFIG);
 
+# silent exit unless explicitly requested
+exit(0) unless setting('use_legacy_sshcollector');
+
 if ($opensshdebug){
     $Net::OpenSSH::debug = ~0;
 }

lib/App/Netdisco.pm

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use 5.010_000;
 
-our $VERSION = '2.040007';
+our $VERSION = '2.041001';
 use App::Netdisco::Configuration;
 
 =head1 NAME
@@ -57,9 +57,9 @@ L<Docker images|https://store.docker.com/community/images/netdisco/netdisco> are
 =back
 
 We have several other pages with tips for
-L<alternate deployment scenarios|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
+L<installation tips|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
 L<understanding and troubleshooting Netdisco|https://github.com/netdisco/netdisco/wiki/Troubleshooting>,
-L<tips and tricks for specific platforms|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
+L<notes for specific device vendors|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
 and L<all the configuration options|https://github.com/netdisco/netdisco/wiki/Configuration>.
 
 You can also speak to someone in the C<#netdisco@freenode> IRC channel, or on
@@ -70,16 +70,19 @@ L<Release Notes|https://github.com/netdisco/netdisco/wiki/Release-Notes>.
 =head1 Dependencies
 
 Netdisco has several Perl library dependencies which will be automatically
-installed. However it's I<strongly> recommended that you first install
+installed. However it's required that you first install the following
-L<DBD::Pg>, L<SNMP>, and a compiler using your operating system packages.
+operating system packages:
 
 On Ubuntu/Debian:
 
- root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl build-essential
+ root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl curl postgresql build-essential
 
 On Fedora/Red-Hat:
 
- root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel make automake gcc
+ root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel curl postgresql-server postgresql-contrib make automake gcc
+ root:~# postgresql-setup initdb
+ root:~# systemctl start postgresql
+ root:~# systemctl enable postgresql
 
 On BSD systems please see L<our BSD tips|https://github.com/netdisco/netdisco/wiki/BSD-Install>.
 
@@ -104,9 +107,11 @@ application:
   
  postgres:~$ createdb -O netdisco netdisco
 
-The default PostgreSQL configuration isn't well tuned for modern server
+You may wish to L<amend the PostgreSQL
-hardware. We strongly recommend that you use the C<pgtune> Python program to
+configuration|https://github.com/netdisco/netdisco/wiki/Install-Tips#enable-md5-authentication-to-postgresql>
-auto-tune your C<postgresql.conf> file:
+so that local connections are working.  The default PostgreSQL configuration
+also needs tuning for modern server hardware. We recommend that you use the
+C<pgtune> Python program to auto-tune your C<postgresql.conf> file:
 
 =over 4
 

lib/App/Netdisco/Backend/Job.pm

@@ -93,7 +93,7 @@ phase.
 
 sub finalise_status {
   my $job = shift;
-  # use DDP; p $job->_statuslist;
+  # use DDP; p $job->_statuslist;
 
   # fallback
   $job->status('error');
@@ -103,7 +103,7 @@ sub finalise_status {
 
   foreach my $status (reverse @{ $job->_statuslist }) {
     next if $status->phase 
-      and $status->phase !~ m/^(?:check|early|main)$/;
+      and $status->phase !~ m/^(?:check|early|main|store|late)$/;
 
     # done() from check phase should not be the action's done()
     next if $status->phase eq 'check' and $status->is_ok;

lib/App/Netdisco/Configuration.pm

@@ -85,6 +85,7 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') })
   config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ];
 }
 # fix up device_auth (or create it from old snmp_auth and community settings)
+# also imports legacy sshcollcetor config
 config->{'device_auth'}
   = [ App::Netdisco::Util::DeviceAuth::fixup_device_auth() ];
 

lib/App/Netdisco/Transport/SNMP.pm

@@ -25,7 +25,7 @@ App::Netdisco::Transport::SNMP
 Singleton for SNMP connections. Returns cached L<SNMP::Info> instance for a
 given device IP, or else undef. All methods are class methods, for example:
 
- App::Netdisco::Transport::SNMP->reader_for( ... );
+ my $snmp = App::Netdisco::Transport::SNMP->reader_for( ... );
 
 =cut
 

lib/App/Netdisco/Transport/SSH.pm

@@ -0,0 +1,115 @@
+package App::Netdisco::Transport::SSH;
+
+use Dancer qw/:syntax :script/;
+
+use App::Netdisco::Util::Device 'get_device';
+use Module::Load ();
+use Net::OpenSSH;
+use Try::Tiny;
+
+use base 'Dancer::Object::Singleton';
+
+=head1 NAME
+
+App::Netdisco::Transport::SSH
+
+=head1 DESCRIPTION
+
+Returns an object which has an active SSH connection which can be used
+for some actions such as arpnip.
+
+ my $cli = App::Netdisco::Transport::SSH->session_for( ... );
+
+=cut
+
+__PACKAGE__->attributes(qw/ sessions /);
+
+sub init {
+  my ( $class, $self ) = @_;
+  $self->sessions( {} );
+  return $self;
+}
+
+=head1 session_for( $ip )
+
+Given an IP address, returns an object instance configured for and connected
+to that device.
+
+Returns C<undef> if the connection fails.
+
+=cut
+
+{
+  package MySession;
+  use Moo;
+
+  has 'ssh'  => ( is => 'rw' );
+  has 'auth' => ( is => 'rw' );
+  has 'host' => ( is => 'rw' );
+  has 'platform' => ( is => 'rw' );
+
+  sub arpnip {
+    my $self = shift;
+    $self->platform->arpnip(@_, $self->host, $self->ssh, $self->auth);
+  }
+}
+
+sub session_for {
+  my ($class, $ip) = @_;
+
+  my $device = get_device($ip) or return undef;
+  my $sessions = $class->instance->sessions or return undef;
+
+  return $sessions->{$device->ip} if exists $sessions->{$device->ip};
+  debug sprintf 'cli session cache warm: [%s]', $device->ip;
+
+  my $auth = (setting('device_auth') || []);
+  if (1 != scalar @$auth) {
+    error sprintf " [%s] require only one matching auth stanza", $device->ip;
+    return undef;
+  }
+  $auth = $auth->[0];
+
+  my @master_opts = qw(-o BatchMode=no);
+  push(@master_opts, @{$auth->{ssh_master_opts}})
+    if $auth->{ssh_master_opts};
+
+  $Net::OpenSSH::debug = $ENV{SSH_TRACE};
+  my $ssh = Net::OpenSSH->new(
+    $device->ip,
+    user => $auth->{username},
+    password => $auth->{password},
+    timeout => 30,
+    async => 0,
+    default_stderr_file => '/dev/null',
+    master_opts => \@master_opts
+  );
+
+  if ($ssh->error) {
+    error sprintf " [%s] ssh connection error [%s]", $device->ip, $ssh->error;
+    return undef;
+  }
+  elsif (! $ssh) {
+    error sprintf " [%s] Net::OpenSSH instantiation error", $device->ip;
+    return undef;
+  }
+
+  my $platform = "App::Netdisco::SSHCollector::Platform::" . $auth->{platform};
+  my $happy = false;
+  try {
+    Module::Load::load $platform;
+    $happy = true;
+  } catch { error $_ };
+  return unless $happy;
+
+  my $sess = MySession->new(
+    ssh  => $ssh,
+    auth => $auth,
+    host => $device->ip,
+    platform => $platform->new(),
+  );
+
+  return ($sessions->{$device->ip} = $sess);
+}
+
+true;

lib/App/Netdisco/Util/DeviceAuth.pm

@@ -63,7 +63,23 @@ sub fixup_device_auth {
     die "error: config: stanza in device_auth must have a tag\n"
       if not $stanza->{tag} and exists $stanza->{user};
 
-    push @new_stanzas, $stanza
+    push @new_stanzas, $stanza;
+  }
+
+  # import legacy sshcollector configuration
+  my $sshcollector = (setting('sshcollector') || []);
+  foreach my $stanza (@$sshcollector) {
+    # defaults
+    $stanza->{driver} = 'cli';
+    $stanza->{read} = 1;
+    $stanza->{no}   ||= [];
+
+    # fixups
+    $stanza->{only} ||= [ scalar delete $stanza->{ip} ||
+                          scalar delete $stanza->{hostname} ];
+    $stanza->{username} = scalar delete $stanza->{user};
+
+    push @new_stanzas, $stanza;
   }
 
   # legacy config 

lib/App/Netdisco/Util/Port.pm

@@ -219,7 +219,7 @@ Returns true if the C<$port> L<DBIx::Class> object has a phone connected.
 =cut
 
 sub port_has_phone {
-  return (shift)->with_properties->remote_is_phone;
+  return (shift)->properties->remote_is_phone;
 }
 
 1;

lib/App/Netdisco/Util/SNMP.pm

@@ -100,11 +100,19 @@ sub snmp_comm_reindex {
       }
       $prefix ||= 'vlan-';
 
-      debug
+      if ($vlan =~ /^[0-9]+$/i && $vlan) { 
-        sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
+        debug sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
         $device->ip, $prefix, $vlan, $ver, $snmp->class;
-      $vlan ? $snmp->update(Context => ($prefix . $vlan))
+        $snmp->update(Context => ($prefix . $vlan));
-            : $snmp->update(Context => '');
+      } elsif ($vlan =~ /^[a-z0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s" (ver: %s, class: %s)',
+          $device->ip, $vlan, $ver, $snmp->class;
+        $snmp->update(Context => ($vlan));
+      } else { 
+        debug sprintf '[%s] reindexing without context (ver: %s, class: %s)',
+          $device->ip, $ver, $snmp->class;
+        $snmp->update(Context => ''); 
+      }
   }
   else {
       my $comm = $snmp->snmp_comm;

lib/App/Netdisco/Worker/Loader.pm

@@ -12,7 +12,9 @@ use namespace::clean;
 has [qw/workers_check
         workers_early
         workers_main
-        workers_user/] => ( is => 'rw' );
+        workers_user
+        workers_store
+        workers_late/] => ( is => 'rw' );
 
 sub load_workers {
   my $self = shift;
@@ -37,7 +39,7 @@ sub load_workers {
   my $workers = vars->{'workers'}->{$action} || {};
   #use DDP; p vars->{'workers'};
 
-  foreach my $phase (qw/check early main user/) {
+  foreach my $phase (qw/check early main user store late/) {
     my $pname = "workers_${phase}";
     my @wset = ();
 

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -1,15 +1,43 @@
 package App::Netdisco::Worker::Plugin::Arpnip::Nodes;
 
 use Dancer ':syntax';
+use Dancer::Plugin::DBIC 'schema';
+
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
+use App::Netdisco::Transport::SSH ();
 use App::Netdisco::Transport::SNMP ();
+
 use App::Netdisco::Util::Node qw/check_mac store_arp/;
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
-use Dancer::Plugin::DBIC 'schema';
+
+use NetAddr::IP::Lite ':lower';
 use Time::HiRes 'gettimeofday';
 
+register_worker({ phase => 'store' }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  # would be possible just to use now() on updated records, but by using this
+  # same value for them all, we _can_ if we want add a job at the end to
+  # select and do something with the updated set (no reason to yet, though)
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  # update node_ip with ARP and Neighbor Cache entries
+
+  store_arp(\%$_, $now) for @{ vars->{'v4arps'} };
+  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
+    $device->ip, scalar @{ vars->{'v4arps'} };
+
+  store_arp(\%$_, $now) for @{ vars->{'v6arps'} };
+  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
+    $device->ip, scalar @{ vars->{'v6arps'} };
+
+  $device->update({last_arpnip => \$now});
+  return Status->done("Ended arpnip for $device");
+});
+
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my ($job, $workerconf) = @_;
 
@@ -17,31 +45,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
     or return Status->defer("arpnip failed: could not SNMP connect to $device");
 
-  # get v4 arp table
+  # cache v4 arp table
-  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
+  push @{ vars->{'v4arps'} },
-  # get v6 neighbor cache
+    @{ get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr) };
-  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
 
-  # would be possible just to use now() on updated records, but by using this
+  # cache v6 neighbor cache
-  # same value for them all, we _can_ if we want add a job at the end to
+  push @{ vars->{'v6arps'} },
-  # select and do something with the updated set (no reason to yet, though)
+    @{get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr) };
-  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
 
-  # update node_ip with ARP and Neighbor Cache entries
+  return Status->info("Gathered arp caches from $device");
-  store_arp(\%$_, $now) for @$v4;
-  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
-    $device->ip, scalar @$v4;
-
-  store_arp(\%$_, $now) for @$v6;
-  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
-    $device->ip, scalar @$v6;
-
-  $device->update({last_arpnip => \$now});
-  return Status->done("Ended arpnip for $device");
 });
 
 # get an arp table (v4 or v6)
-sub get_arps {
+sub get_arps_snmp {
   my ($device, $paddr, $netaddr) = @_;
   my @arps = ();
 
@@ -63,4 +79,46 @@ sub get_arps {
   return $resolved_ips;
 }
 
+register_worker({ phase => 'main', driver => 'cli' }, sub {
+    my ($job, $workerconf) = @_;
+
+    my $device = $job->device;
+    my $cli = App::Netdisco::Transport::SSH->session_for($device)
+      or return Status->defer("arpnip failed: could not SSH connect to $device");
+
+    # should be both v4 and v6
+    my @arps = @{ get_arps_cli($device, [$cli->arpnip]) };
+
+    # cache v4 arp table
+    push @{ vars->{'v4arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits ==  32 } @arps;
+
+    # cache v6 neighbor cache
+    push @{ vars->{'v6arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits == 128 } @arps;
+
+    return Status->info("Gathered arp caches from $device");
+});
+
+sub get_arps_cli {
+  my ($device, $entries) = @_;
+  my @arps = ();
+  $entries ||= [];
+
+  foreach my $entry (@$entries) {
+    next unless check_mac($entry->{mac}, $device);
+    push @arps, {
+        node => $entry->{mac},
+        ip   => $entry->{ip},
+        dns  => $entry->{dns},
+    };
+  }
+
+  debug sprintf ' resolving %d ARP entries with max %d outstanding requests',
+    scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_ips = hostnames_resolve_async(\@arps);
+
+  return $resolved_ips;
+}
+
 true;

lib/App/Netdisco/Worker/Runner.pm

@@ -69,7 +69,7 @@ sub run {
 
     # run other phases
     if ($job->check_passed) {
-      $self->run_workers("workers_${_}") for qw/early main user/;
+      $self->run_workers("workers_${_}") for qw/early main user store late/;
     }
   };
 

share/config.yml

@@ -217,6 +217,7 @@ device_identity: []
 community: []
 community_rw: []
 device_auth: []
+use_legacy_sshcollector: false
 get_credentials: ""
 bulkwalk_off: false
 bulkwalk_no: []
@@ -416,7 +417,6 @@ worker_plugins:
   - 'Vlan::Core'
 
 extra_worker_plugins: []
-#  - Discover::ConfigBackup::CLI
 
 driver_priority:
   restconf: 500

xt/00-compile.t

@@ -21,8 +21,8 @@ use Test::Compile;
 
 my $test = Test::Compile->new();
 
-my @plfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pl_files();
+my @plfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pl_files();
-my @pmfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pm_files();
+my @pmfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pm_files();
 
 $test->ok($test->pl_file_compiles($_), "$_ compiles") for @plfiles;
 $test->ok($test->pm_file_compiles($_), "$_ compiles") for @pmfiles;