Merge branch 'master' into og-api-tokens-simple

2019-03-15 12:31:52 +00:00
parent 25d46c0b40 9b947369f9
commit 20f4704fe2
19 changed files with 309 additions and 59 deletions
--- a/Build.PL
+++ b/Build.PL
@@ -38,6 +38,7 @@ Module::Build->new(
    'Dancer::Plugin::Passphrase' => '2.0.1',
    'Dancer::Plugin::Swagger' => '0',
    'Dancer::Session::Cookie' => '0.27',
+    'Expect' => '0',
    'File::ShareDir' => '1.03',
    'File::Slurper' => '0.009',
    'Guard' => '1.022',
@@ -55,6 +56,7 @@ Module::Build->new(
    'Net::Domain' => '1.23',
    'Net::DNS' => '0.72',
    'Net::LDAP' => '0',
+    'Net::OpenSSH' => '0',
    'NetAddr::MAC' => '0.93',
    'NetAddr::IP' => '4.068',
    'Opcode' => '1.07',
@@ -91,8 +93,6 @@ Module::Build->new(
  recommends => {
    'Graph' => '0',
    'GraphViz' => '0',
-    'Net::OpenSSH' => '0',
-    'Expect' => '0',
  },
  test_requires => {
    'Test::More' => '1.302083',
--- a/22
+++ b/22
@@ -1,3 +1,25 @@
+2.041001 - 2019-03-15
+
+  [ENHANCEMENTS]
+
+  * add "store" and "late" phases to backend workers
+  * documentation updates
+
+  [BUG FIXES]
+
+  * #539 fix an issue with VLAN reindixing for VRFs (earendilfr)
+  * fix device port change check
+
+2.041000 - 2019-03-12
+
+  [NEW FEATURES]
+
+  * netdisco-sshcollector no longer required - the functionality (arpnip via
+    ssh) will be run within netdisco's core schedule (rc9000 and ollyg)
+
+  * get_credentials replaces get_community and accepts any device_auth stanza
+    in JSON format (ollyg)
+
 2.040007 - 2019-03-06

  [BUG FIXES]
--- a/3
+++ b/3
@@ -110,7 +110,9 @@ lib/App/Netdisco/SSHCollector/Platform/Linux.pm
 lib/App/Netdisco/SSHCollector/Platform/NXOS.pm
 lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
 lib/App/Netdisco/Transport/SNMP.pm
+lib/App/Netdisco/Transport/SSH.pm
 lib/App/Netdisco/Util/Device.pm
+lib/App/Netdisco/Util/DeviceAuth.pm
 lib/App/Netdisco/Util/DNS.pm
 lib/App/Netdisco/Util/ExpandParams.pm
 lib/App/Netdisco/Util/FastResolver.pm
@@ -485,6 +487,7 @@ share/views/sidebar/report/subnets.tt
 share/views/sidebar/search/device.tt
 share/views/sidebar/search/node.tt
 share/views/sidebar/search/port.tt
+xt/00-compile.t
 xt/10-sort_port.t
 xt/11-portsort.t
 xt/20-checkacl.t
--- a/META.json
+++ b/META.json
@@ -29,10 +29,8 @@
      },
      "runtime" : {
         "recommends" : {
-            "Expect" : "0",
            "Graph" : "0",
-            "GraphViz" : "0",
-            "Net::OpenSSH" : "0"
+            "GraphViz" : "0"
         },
         "requires" : {
            "Algorithm::Cron" : "0.07",
@@ -52,6 +50,7 @@
            "Dancer::Plugin::Passphrase" : "v2.0.1",
            "Dancer::Session::Cookie" : "0.27",
            "Data::Printer" : "0",
+            "Expect" : "0",
            "File::ShareDir" : "1.03",
            "File::Slurper" : "0.009",
            "Guard" : "1.022",
@@ -69,6 +68,7 @@
            "Net::DNS" : "0.72",
            "Net::Domain" : "1.23",
            "Net::LDAP" : "0",
+            "Net::OpenSSH" : "0",
            "NetAddr::IP" : "4.068",
            "NetAddr::MAC" : "0.93",
            "Opcode" : "1.07",
@@ -110,6 +110,7 @@
      "test" : {
         "requires" : {
            "Env::Path" : "0",
+            "Test::Compile" : "0",
            "Test::File::ShareDir::Dist" : "0",
            "Test::More" : "1.302083"
         }
@@ -118,7 +119,7 @@
   "provides" : {
      "App::Netdisco" : {
         "file" : "lib/App/Netdisco.pm",
-         "version" : "2.040007"
+         "version" : "2.041001"
      },
      "App::Netdisco::AnyEvent::Nbtstat" : {
         "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm"
@@ -409,12 +410,18 @@
      "App::Netdisco::Transport::SNMP" : {
         "file" : "lib/App/Netdisco/Transport/SNMP.pm"
      },
+      "App::Netdisco::Transport::SSH" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
+      },
      "App::Netdisco::Util::DNS" : {
         "file" : "lib/App/Netdisco/Util/DNS.pm"
      },
      "App::Netdisco::Util::Device" : {
         "file" : "lib/App/Netdisco/Util/Device.pm"
      },
+      "App::Netdisco::Util::DeviceAuth" : {
+         "file" : "lib/App/Netdisco/Util/DeviceAuth.pm"
+      },
      "App::Netdisco::Util::ExpandParams" : {
         "file" : "lib/App/Netdisco/Util/ExpandParams.pm"
      },
@@ -783,6 +790,9 @@
      },
      "Dancer::Template::NetdiscoTemplateToolkit" : {
         "file" : "lib/Dancer/Template/NetdiscoTemplateToolkit.pm"
+      },
+      "MySession" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
      }
   },
   "release_status" : "stable",
@@ -792,7 +802,7 @@
      },
      "homepage" : "http://netdisco.org/",
      "license" : [
-         "http://opensource.org/licenses/bsd-license.php"
+         "http://opensource.org/licenses/BSD-3-Clause"
      ],
      "repository" : {
         "url" : "https://github.com/netdisco/netdisco"
@@ -800,6 +810,6 @@
      "x_IRC" : "irc://irc.freenode.org/#netdisco",
      "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users"
   },
-   "version" : "2.040007",
+   "version" : "2.041001",
   "x_serialization_backend" : "JSON::PP version 2.97001"
 }
--- a/META.yml
+++ b/META.yml
@@ -7,6 +7,7 @@ build_requires:
  ExtUtils::Config: '0'
  ExtUtils::Helpers: '0'
  ExtUtils::InstallPaths: '0'
+  Test::Compile: '0'
  Test::File::ShareDir::Dist: '0'
  Test::More: '1.302083'
 configure_requires:
@@ -22,7 +23,7 @@ name: App-Netdisco
 provides:
  App::Netdisco:
    file: lib/App/Netdisco.pm
-    version: '2.040007'
+    version: '2.041001'
  App::Netdisco::AnyEvent::Nbtstat:
    file: lib/App/Netdisco/AnyEvent/Nbtstat.pm
  App::Netdisco::Backend::Job:
@@ -216,10 +217,14 @@ provides:
    file: lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
  App::Netdisco::Transport::SNMP:
    file: lib/App/Netdisco/Transport/SNMP.pm
+  App::Netdisco::Transport::SSH:
+    file: lib/App/Netdisco/Transport/SSH.pm
  App::Netdisco::Util::DNS:
    file: lib/App/Netdisco/Util/DNS.pm
  App::Netdisco::Util::Device:
    file: lib/App/Netdisco/Util/Device.pm
+  App::Netdisco::Util::DeviceAuth:
+    file: lib/App/Netdisco/Util/DeviceAuth.pm
  App::Netdisco::Util::ExpandParams:
    file: lib/App/Netdisco/Util/ExpandParams.pm
  App::Netdisco::Util::FastResolver:
@@ -466,11 +471,11 @@ provides:
    file: lib/App/Netdisco/Worker/Status.pm
  Dancer::Template::NetdiscoTemplateToolkit:
    file: lib/Dancer/Template/NetdiscoTemplateToolkit.pm
+  MySession:
+    file: lib/App/Netdisco/Transport/SSH.pm
 recommends:
-  Expect: '0'
  Graph: '0'
  GraphViz: '0'
-  Net::OpenSSH: '0'
 requires:
  Algorithm::Cron: '0.07'
  AnyEvent: '7.05'
@@ -489,6 +494,7 @@ requires:
  Dancer::Plugin::Passphrase: v2.0.1
  Dancer::Session::Cookie: '0.27'
  Data::Printer: '0'
+  Expect: '0'
  File::ShareDir: '1.03'
  File::Slurper: '0.009'
  Guard: '1.022'
@@ -506,6 +512,7 @@ requires:
  Net::DNS: '0.72'
  Net::Domain: '1.23'
  Net::LDAP: '0'
+  Net::OpenSSH: '0'
  NetAddr::IP: '4.068'
  NetAddr::MAC: '0.93'
  Opcode: '1.07'
@@ -547,7 +554,7 @@ resources:
  MailingList: https://lists.sourceforge.net/lists/listinfo/netdisco-users
  bugtracker: https://github.com/netdisco/netdisco/issues
  homepage: http://netdisco.org/
-  license: http://opensource.org/licenses/bsd-license.php
+  license: http://opensource.org/licenses/BSD-3-Clause
  repository: https://github.com/netdisco/netdisco
-version: '2.040007'
+version: '2.041001'
 x_serialization_backend: 'CPAN::Meta::YAML version 0.018'
--- a/bin/netdisco-sshcollector
+++ b/bin/netdisco-sshcollector
@@ -74,6 +74,9 @@ $ENV{DBIC_TRACE} ||= $sqltrace;
 # reconfigure logging to force console output
 Dancer::Logger->init('console', $CONFIG);

+# silent exit unless explicitly requested
+exit(0) unless setting('use_legacy_sshcollector');
+
 if ($opensshdebug){
    $Net::OpenSSH::debug = ~0;
 }
--- a/lib/App/Netdisco.pm
+++ b/lib/App/Netdisco.pm
@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use 5.010_000;

-our $VERSION = '2.040007';
+our $VERSION = '2.041001';
 use App::Netdisco::Configuration;

 =head1 NAME
@@ -57,9 +57,9 @@ L<Docker images|https://store.docker.com/community/images/netdisco/netdisco> are
 =back

 We have several other pages with tips for
-L<alternate deployment scenarios|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
+L<installation tips|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
 L<understanding and troubleshooting Netdisco|https://github.com/netdisco/netdisco/wiki/Troubleshooting>,
-L<tips and tricks for specific platforms|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
+L<notes for specific device vendors|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
 and L<all the configuration options|https://github.com/netdisco/netdisco/wiki/Configuration>.

 You can also speak to someone in the C<#netdisco@freenode> IRC channel, or on
@@ -70,16 +70,19 @@ L<Release Notes|https://github.com/netdisco/netdisco/wiki/Release-Notes>.
 =head1 Dependencies

 Netdisco has several Perl library dependencies which will be automatically
-installed. However it's I<strongly> recommended that you first install
-L<DBD::Pg>, L<SNMP>, and a compiler using your operating system packages.
+installed. However it's required that you first install the following
+operating system packages:

 On Ubuntu/Debian:

- root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl build-essential
+ root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl curl postgresql build-essential

 On Fedora/Red-Hat:

- root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel make automake gcc
+ root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel curl postgresql-server postgresql-contrib make automake gcc
+ root:~# postgresql-setup initdb
+ root:~# systemctl start postgresql
+ root:~# systemctl enable postgresql

 On BSD systems please see L<our BSD tips|https://github.com/netdisco/netdisco/wiki/BSD-Install>.

@@ -104,9 +107,11 @@ application:
  
 postgres:~$ createdb -O netdisco netdisco

-The default PostgreSQL configuration isn't well tuned for modern server
-hardware. We strongly recommend that you use the C<pgtune> Python program to
-auto-tune your C<postgresql.conf> file:
+You may wish to L<amend the PostgreSQL
+configuration|https://github.com/netdisco/netdisco/wiki/Install-Tips#enable-md5-authentication-to-postgresql>
+so that local connections are working.  The default PostgreSQL configuration
+also needs tuning for modern server hardware. We recommend that you use the
+C<pgtune> Python program to auto-tune your C<postgresql.conf> file:

 =over 4

--- a/lib/App/Netdisco/Backend/Job.pm
+++ b/lib/App/Netdisco/Backend/Job.pm
@@ -93,7 +93,7 @@ phase.

 sub finalise_status {
  my $job = shift;
-  # use DDP; p $job->_statuslist;
+  # use DDP; p $job->_statuslist;

  # fallback
  $job->status('error');
@@ -103,7 +103,7 @@ sub finalise_status {

  foreach my $status (reverse @{ $job->_statuslist }) {
    next if $status->phase 
-      and $status->phase !~ m/^(?:check|early|main)$/;
+      and $status->phase !~ m/^(?:check|early|main|store|late)$/;

    # done() from check phase should not be the action's done()
    next if $status->phase eq 'check' and $status->is_ok;
--- a/lib/App/Netdisco/Configuration.pm
+++ b/lib/App/Netdisco/Configuration.pm
@@ -85,6 +85,7 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') })
  config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ];
 }
 # fix up device_auth (or create it from old snmp_auth and community settings)
+# also imports legacy sshcollcetor config
 config->{'device_auth'}
  = [ App::Netdisco::Util::DeviceAuth::fixup_device_auth() ];

--- a/lib/App/Netdisco/Transport/SNMP.pm
+++ b/lib/App/Netdisco/Transport/SNMP.pm
@@ -25,7 +25,7 @@ App::Netdisco::Transport::SNMP
 Singleton for SNMP connections. Returns cached L<SNMP::Info> instance for a
 given device IP, or else undef. All methods are class methods, for example:

- App::Netdisco::Transport::SNMP->reader_for( ... );
+ my $snmp = App::Netdisco::Transport::SNMP->reader_for( ... );

 =cut

--- a/lib/App/Netdisco/Transport/SSH.pm
+++ b/lib/App/Netdisco/Transport/SSH.pm
@@ -0,0 +1,115 @@
+package App::Netdisco::Transport::SSH;
+
+use Dancer qw/:syntax :script/;
+
+use App::Netdisco::Util::Device 'get_device';
+use Module::Load ();
+use Net::OpenSSH;
+use Try::Tiny;
+
+use base 'Dancer::Object::Singleton';
+
+=head1 NAME
+
+App::Netdisco::Transport::SSH
+
+=head1 DESCRIPTION
+
+Returns an object which has an active SSH connection which can be used
+for some actions such as arpnip.
+
+ my $cli = App::Netdisco::Transport::SSH->session_for( ... );
+
+=cut
+
+__PACKAGE__->attributes(qw/ sessions /);
+
+sub init {
+  my ( $class, $self ) = @_;
+  $self->sessions( {} );
+  return $self;
+}
+
+=head1 session_for( $ip )
+
+Given an IP address, returns an object instance configured for and connected
+to that device.
+
+Returns C<undef> if the connection fails.
+
+=cut
+
+{
+  package MySession;
+  use Moo;
+
+  has 'ssh'  => ( is => 'rw' );
+  has 'auth' => ( is => 'rw' );
+  has 'host' => ( is => 'rw' );
+  has 'platform' => ( is => 'rw' );
+
+  sub arpnip {
+    my $self = shift;
+    $self->platform->arpnip(@_, $self->host, $self->ssh, $self->auth);
+  }
+}
+
+sub session_for {
+  my ($class, $ip) = @_;
+
+  my $device = get_device($ip) or return undef;
+  my $sessions = $class->instance->sessions or return undef;
+
+  return $sessions->{$device->ip} if exists $sessions->{$device->ip};
+  debug sprintf 'cli session cache warm: [%s]', $device->ip;
+
+  my $auth = (setting('device_auth') || []);
+  if (1 != scalar @$auth) {
+    error sprintf " [%s] require only one matching auth stanza", $device->ip;
+    return undef;
+  }
+  $auth = $auth->[0];
+
+  my @master_opts = qw(-o BatchMode=no);
+  push(@master_opts, @{$auth->{ssh_master_opts}})
+    if $auth->{ssh_master_opts};
+
+  $Net::OpenSSH::debug = $ENV{SSH_TRACE};
+  my $ssh = Net::OpenSSH->new(
+    $device->ip,
+    user => $auth->{username},
+    password => $auth->{password},
+    timeout => 30,
+    async => 0,
+    default_stderr_file => '/dev/null',
+    master_opts => \@master_opts
+  );
+
+  if ($ssh->error) {
+    error sprintf " [%s] ssh connection error [%s]", $device->ip, $ssh->error;
+    return undef;
+  }
+  elsif (! $ssh) {
+    error sprintf " [%s] Net::OpenSSH instantiation error", $device->ip;
+    return undef;
+  }
+
+  my $platform = "App::Netdisco::SSHCollector::Platform::" . $auth->{platform};
+  my $happy = false;
+  try {
+    Module::Load::load $platform;
+    $happy = true;
+  } catch { error $_ };
+  return unless $happy;
+
+  my $sess = MySession->new(
+    ssh  => $ssh,
+    auth => $auth,
+    host => $device->ip,
+    platform => $platform->new(),
+  );
+
+  return ($sessions->{$device->ip} = $sess);
+}
+
+true;
--- a/lib/App/Netdisco/Util/DeviceAuth.pm
+++ b/lib/App/Netdisco/Util/DeviceAuth.pm
@@ -63,7 +63,23 @@ sub fixup_device_auth {
    die "error: config: stanza in device_auth must have a tag\n"
      if not $stanza->{tag} and exists $stanza->{user};

-    push @new_stanzas, $stanza
+    push @new_stanzas, $stanza;
+  }
+
+  # import legacy sshcollector configuration
+  my $sshcollector = (setting('sshcollector') || []);
+  foreach my $stanza (@$sshcollector) {
+    # defaults
+    $stanza->{driver} = 'cli';
+    $stanza->{read} = 1;
+    $stanza->{no}   ||= [];
+
+    # fixups
+    $stanza->{only} ||= [ scalar delete $stanza->{ip} ||
+                          scalar delete $stanza->{hostname} ];
+    $stanza->{username} = scalar delete $stanza->{user};
+
+    push @new_stanzas, $stanza;
  }

  # legacy config 
--- a/lib/App/Netdisco/Util/Port.pm
+++ b/lib/App/Netdisco/Util/Port.pm
@@ -219,7 +219,7 @@ Returns true if the C<$port> L<DBIx::Class> object has a phone connected.
 =cut

 sub port_has_phone {
-  return (shift)->with_properties->remote_is_phone;
+  return (shift)->properties->remote_is_phone;
 }

 1;
--- a/lib/App/Netdisco/Util/SNMP.pm
+++ b/lib/App/Netdisco/Util/SNMP.pm
@@ -100,11 +100,19 @@ sub snmp_comm_reindex {
      }
      $prefix ||= 'vlan-';

-      debug
-        sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
+      if ($vlan =~ /^[0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
        $device->ip, $prefix, $vlan, $ver, $snmp->class;
-      $vlan ? $snmp->update(Context => ($prefix . $vlan))
-            : $snmp->update(Context => '');
+        $snmp->update(Context => ($prefix . $vlan));
+      } elsif ($vlan =~ /^[a-z0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s" (ver: %s, class: %s)',
+          $device->ip, $vlan, $ver, $snmp->class;
+        $snmp->update(Context => ($vlan));
+      } else { 
+        debug sprintf '[%s] reindexing without context (ver: %s, class: %s)',
+          $device->ip, $ver, $snmp->class;
+        $snmp->update(Context => ''); 
+      }
  }
  else {
      my $comm = $snmp->snmp_comm;
--- a/lib/App/Netdisco/Worker/Loader.pm
+++ b/lib/App/Netdisco/Worker/Loader.pm
@@ -12,7 +12,9 @@ use namespace::clean;
 has [qw/workers_check
        workers_early
        workers_main
-        workers_user/] => ( is => 'rw' );
+        workers_user
+        workers_store
+        workers_late/] => ( is => 'rw' );

 sub load_workers {
  my $self = shift;
@@ -37,7 +39,7 @@ sub load_workers {
  my $workers = vars->{'workers'}->{$action} || {};
  #use DDP; p vars->{'workers'};

-  foreach my $phase (qw/check early main user/) {
+  foreach my $phase (qw/check early main user store late/) {
    my $pname = "workers_${phase}";
    my @wset = ();

--- a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
+++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
@@ -1,15 +1,43 @@
 package App::Netdisco::Worker::Plugin::Arpnip::Nodes;

 use Dancer ':syntax';
+use Dancer::Plugin::DBIC 'schema';
+
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';

+use App::Netdisco::Transport::SSH ();
 use App::Netdisco::Transport::SNMP ();
+
 use App::Netdisco::Util::Node qw/check_mac store_arp/;
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
-use Dancer::Plugin::DBIC 'schema';
+
+use NetAddr::IP::Lite ':lower';
 use Time::HiRes 'gettimeofday';

+register_worker({ phase => 'store' }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  # would be possible just to use now() on updated records, but by using this
+  # same value for them all, we _can_ if we want add a job at the end to
+  # select and do something with the updated set (no reason to yet, though)
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  # update node_ip with ARP and Neighbor Cache entries
+
+  store_arp(\%$_, $now) for @{ vars->{'v4arps'} };
+  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
+    $device->ip, scalar @{ vars->{'v4arps'} };
+
+  store_arp(\%$_, $now) for @{ vars->{'v6arps'} };
+  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
+    $device->ip, scalar @{ vars->{'v6arps'} };
+
+  $device->update({last_arpnip => \$now});
+  return Status->done("Ended arpnip for $device");
+});
+
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
  my ($job, $workerconf) = @_;

@@ -17,31 +45,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
    or return Status->defer("arpnip failed: could not SNMP connect to $device");

-  # get v4 arp table
-  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
-  # get v6 neighbor cache
-  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
+  # cache v4 arp table
+  push @{ vars->{'v4arps'} },
+    @{ get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr) };

-  # would be possible just to use now() on updated records, but by using this
-  # same value for them all, we _can_ if we want add a job at the end to
-  # select and do something with the updated set (no reason to yet, though)
-  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+  # cache v6 neighbor cache
+  push @{ vars->{'v6arps'} },
+    @{get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr) };

-  # update node_ip with ARP and Neighbor Cache entries
-  store_arp(\%$_, $now) for @$v4;
-  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
-    $device->ip, scalar @$v4;
-
-  store_arp(\%$_, $now) for @$v6;
-  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
-    $device->ip, scalar @$v6;
-
-  $device->update({last_arpnip => \$now});
-  return Status->done("Ended arpnip for $device");
+  return Status->info("Gathered arp caches from $device");
 });

 # get an arp table (v4 or v6)
-sub get_arps {
+sub get_arps_snmp {
  my ($device, $paddr, $netaddr) = @_;
  my @arps = ();

@@ -63,4 +79,46 @@ sub get_arps {
  return $resolved_ips;
 }

+register_worker({ phase => 'main', driver => 'cli' }, sub {
+    my ($job, $workerconf) = @_;
+
+    my $device = $job->device;
+    my $cli = App::Netdisco::Transport::SSH->session_for($device)
+      or return Status->defer("arpnip failed: could not SSH connect to $device");
+
+    # should be both v4 and v6
+    my @arps = @{ get_arps_cli($device, [$cli->arpnip]) };
+
+    # cache v4 arp table
+    push @{ vars->{'v4arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits ==  32 } @arps;
+
+    # cache v6 neighbor cache
+    push @{ vars->{'v6arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits == 128 } @arps;
+
+    return Status->info("Gathered arp caches from $device");
+});
+
+sub get_arps_cli {
+  my ($device, $entries) = @_;
+  my @arps = ();
+  $entries ||= [];
+
+  foreach my $entry (@$entries) {
+    next unless check_mac($entry->{mac}, $device);
+    push @arps, {
+        node => $entry->{mac},
+        ip   => $entry->{ip},
+        dns  => $entry->{dns},
+    };
+  }
+
+  debug sprintf ' resolving %d ARP entries with max %d outstanding requests',
+    scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_ips = hostnames_resolve_async(\@arps);
+
+  return $resolved_ips;
+}
+
 true;
--- a/lib/App/Netdisco/Worker/Runner.pm
+++ b/lib/App/Netdisco/Worker/Runner.pm
@@ -69,7 +69,7 @@ sub run {

    # run other phases
    if ($job->check_passed) {
-      $self->run_workers("workers_${_}") for qw/early main user/;
+      $self->run_workers("workers_${_}") for qw/early main user store late/;
    }
  };

--- a/share/config.yml
+++ b/share/config.yml
@@ -217,6 +217,7 @@ device_identity: []
 community: []
 community_rw: []
 device_auth: []
+use_legacy_sshcollector: false
 get_credentials: ""
 bulkwalk_off: false
 bulkwalk_no: []
@@ -416,7 +417,6 @@ worker_plugins:
  - 'Vlan::Core'

 extra_worker_plugins: []
-#  - Discover::ConfigBackup::CLI

 driver_priority:
  restconf: 500
--- a/xt/00-compile.t
+++ b/xt/00-compile.t
@@ -21,8 +21,8 @@ use Test::Compile;

 my $test = Test::Compile->new();

-my @plfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pl_files();
-my @pmfiles = grep {$_ !~ m/(?:sshcollector|graph)/i} $test->all_pm_files();
+my @plfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pl_files();
+my @pmfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pm_files();

 $test->ok($test->pl_file_compiles($_), "$_ compiles") for @plfiles;
 $test->ok($test->pm_file_compiles($_), "$_ compiles") for @pmfiles;