Merge branch 'master' of https://github.com/netdisco/netdisco

.travis.yml

@@ -10,7 +10,8 @@ branches:
   only:
     - /^2\.\d{6}$/
     - 'master'
-install: true
+install:
+  - cpanm --quiet --notest PkgConfig Test::CChecker Alien::zlib::Static Alien::OpenSSL::Static Alien::SNMP
 script: |
   perl Build.PL && \
   ./Build && \

Build.PL

@@ -37,6 +37,7 @@ Module::Build->new(
     'Dancer::Plugin::Auth::Extensible' => '0.30',
     'Dancer::Plugin::Passphrase' => '2.0.1',
     'Dancer::Session::Cookie' => '0.27',
+    'Expect' => '0',
     'File::ShareDir' => '1.03',
     'File::Slurper' => '0.009',
     'Guard' => '1.022',
@@ -54,6 +55,7 @@ Module::Build->new(
     'Net::Domain' => '1.23',
     'Net::DNS' => '0.72',
     'Net::LDAP' => '0',
+    'Net::OpenSSH' => '0',
     'NetAddr::MAC' => '0.93',
     'NetAddr::IP' => '4.068',
     'Opcode' => '1.07',
@@ -90,12 +92,11 @@ Module::Build->new(
   recommends => {
     'Graph' => '0',
     'GraphViz' => '0',
-    'Net::OpenSSH' => '0',
-    'Expect' => '0',
   },
   test_requires => {
     'Test::More' => '1.302083',
     'Env::Path'  => '0',
+    'Test::Compile' => '0',
     'Test::File::ShareDir::Dist' => '0',
   },
   script_files => [

Changes

@@ -1,3 +1,25 @@
+2.041001 - 2019-03-15
+
+  [ENHANCEMENTS]
+
+  * add "store" and "late" phases to backend workers
+  * documentation updates
+
+  [BUG FIXES]
+
+  * #539 fix an issue with VLAN reindixing for VRFs (earendilfr)
+  * fix device port change check
+
+2.041000 - 2019-03-12
+
+  [NEW FEATURES]
+
+  * netdisco-sshcollector no longer required - the functionality (arpnip via
+    ssh) will be run within netdisco's core schedule (rc9000 and ollyg)
+
+  * get_credentials replaces get_community and accepts any device_auth stanza
+    in JSON format (ollyg)
+
 2.040007 - 2019-03-06
 
   [BUG FIXES]

MANIFEST

@@ -110,7 +110,9 @@ lib/App/Netdisco/SSHCollector/Platform/Linux.pm
 lib/App/Netdisco/SSHCollector/Platform/NXOS.pm
 lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
 lib/App/Netdisco/Transport/SNMP.pm
+lib/App/Netdisco/Transport/SSH.pm
 lib/App/Netdisco/Util/Device.pm
+lib/App/Netdisco/Util/DeviceAuth.pm
 lib/App/Netdisco/Util/DNS.pm
 lib/App/Netdisco/Util/ExpandParams.pm
 lib/App/Netdisco/Util/FastResolver.pm
@@ -485,6 +487,7 @@ share/views/sidebar/report/subnets.tt
 share/views/sidebar/search/device.tt
 share/views/sidebar/search/node.tt
 share/views/sidebar/search/port.tt
+xt/00-compile.t
 xt/10-sort_port.t
 xt/11-portsort.t
 xt/20-checkacl.t

META.json

@@ -29,10 +29,8 @@
       },
       "runtime" : {
          "recommends" : {
-            "Expect" : "0",
             "Graph" : "0",
-            "GraphViz" : "0",
+            "GraphViz" : "0"
-            "Net::OpenSSH" : "0"
          },
          "requires" : {
             "Algorithm::Cron" : "0.07",
@@ -52,6 +50,7 @@
             "Dancer::Plugin::Passphrase" : "v2.0.1",
             "Dancer::Session::Cookie" : "0.27",
             "Data::Printer" : "0",
+            "Expect" : "0",
             "File::ShareDir" : "1.03",
             "File::Slurper" : "0.009",
             "Guard" : "1.022",
@@ -69,6 +68,7 @@
             "Net::DNS" : "0.72",
             "Net::Domain" : "1.23",
             "Net::LDAP" : "0",
+            "Net::OpenSSH" : "0",
             "NetAddr::IP" : "4.068",
             "NetAddr::MAC" : "0.93",
             "Opcode" : "1.07",
@@ -110,6 +110,7 @@
       "test" : {
          "requires" : {
             "Env::Path" : "0",
+            "Test::Compile" : "0",
             "Test::File::ShareDir::Dist" : "0",
             "Test::More" : "1.302083"
          }
@@ -118,7 +119,7 @@
    "provides" : {
       "App::Netdisco" : {
          "file" : "lib/App/Netdisco.pm",
-         "version" : "2.040007"
+         "version" : "2.041001"
       },
       "App::Netdisco::AnyEvent::Nbtstat" : {
          "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm"
@@ -409,12 +410,18 @@
       "App::Netdisco::Transport::SNMP" : {
          "file" : "lib/App/Netdisco/Transport/SNMP.pm"
       },
+      "App::Netdisco::Transport::SSH" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
+      },
       "App::Netdisco::Util::DNS" : {
          "file" : "lib/App/Netdisco/Util/DNS.pm"
       },
       "App::Netdisco::Util::Device" : {
          "file" : "lib/App/Netdisco/Util/Device.pm"
       },
+      "App::Netdisco::Util::DeviceAuth" : {
+         "file" : "lib/App/Netdisco/Util/DeviceAuth.pm"
+      },
       "App::Netdisco::Util::ExpandParams" : {
          "file" : "lib/App/Netdisco/Util/ExpandParams.pm"
       },
@@ -783,6 +790,9 @@
       },
       "Dancer::Template::NetdiscoTemplateToolkit" : {
          "file" : "lib/Dancer/Template/NetdiscoTemplateToolkit.pm"
+      },
+      "MySession" : {
+         "file" : "lib/App/Netdisco/Transport/SSH.pm"
       }
    },
    "release_status" : "stable",
@@ -792,7 +802,7 @@
       },
       "homepage" : "http://netdisco.org/",
       "license" : [
-         "http://opensource.org/licenses/bsd-license.php"
+         "http://opensource.org/licenses/BSD-3-Clause"
       ],
       "repository" : {
          "url" : "https://github.com/netdisco/netdisco"
@@ -800,6 +810,6 @@
       "x_IRC" : "irc://irc.freenode.org/#netdisco",
       "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users"
    },
-   "version" : "2.040007",
+   "version" : "2.041001",
    "x_serialization_backend" : "JSON::PP version 2.97001"
 }

META.yml

@@ -7,6 +7,7 @@ build_requires:
   ExtUtils::Config: '0'
   ExtUtils::Helpers: '0'
   ExtUtils::InstallPaths: '0'
+  Test::Compile: '0'
   Test::File::ShareDir::Dist: '0'
   Test::More: '1.302083'
 configure_requires:
@@ -22,7 +23,7 @@ name: App-Netdisco
 provides:
   App::Netdisco:
     file: lib/App/Netdisco.pm
-    version: '2.040007'
+    version: '2.041001'
   App::Netdisco::AnyEvent::Nbtstat:
     file: lib/App/Netdisco/AnyEvent/Nbtstat.pm
   App::Netdisco::Backend::Job:
@@ -216,10 +217,14 @@ provides:
     file: lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm
   App::Netdisco::Transport::SNMP:
     file: lib/App/Netdisco/Transport/SNMP.pm
+  App::Netdisco::Transport::SSH:
+    file: lib/App/Netdisco/Transport/SSH.pm
   App::Netdisco::Util::DNS:
     file: lib/App/Netdisco/Util/DNS.pm
   App::Netdisco::Util::Device:
     file: lib/App/Netdisco/Util/Device.pm
+  App::Netdisco::Util::DeviceAuth:
+    file: lib/App/Netdisco/Util/DeviceAuth.pm
   App::Netdisco::Util::ExpandParams:
     file: lib/App/Netdisco/Util/ExpandParams.pm
   App::Netdisco::Util::FastResolver:
@@ -466,11 +471,11 @@ provides:
     file: lib/App/Netdisco/Worker/Status.pm
   Dancer::Template::NetdiscoTemplateToolkit:
     file: lib/Dancer/Template/NetdiscoTemplateToolkit.pm
+  MySession:
+    file: lib/App/Netdisco/Transport/SSH.pm
 recommends:
-  Expect: '0'
   Graph: '0'
   GraphViz: '0'
-  Net::OpenSSH: '0'
 requires:
   Algorithm::Cron: '0.07'
   AnyEvent: '7.05'
@@ -489,6 +494,7 @@ requires:
   Dancer::Plugin::Passphrase: v2.0.1
   Dancer::Session::Cookie: '0.27'
   Data::Printer: '0'
+  Expect: '0'
   File::ShareDir: '1.03'
   File::Slurper: '0.009'
   Guard: '1.022'
@@ -506,6 +512,7 @@ requires:
   Net::DNS: '0.72'
   Net::Domain: '1.23'
   Net::LDAP: '0'
+  Net::OpenSSH: '0'
   NetAddr::IP: '4.068'
   NetAddr::MAC: '0.93'
   Opcode: '1.07'
@@ -547,7 +554,7 @@ resources:
   MailingList: https://lists.sourceforge.net/lists/listinfo/netdisco-users
   bugtracker: https://github.com/netdisco/netdisco/issues
   homepage: http://netdisco.org/
-  license: http://opensource.org/licenses/bsd-license.php
+  license: http://opensource.org/licenses/BSD-3-Clause
   repository: https://github.com/netdisco/netdisco
-version: '2.040007'
+version: '2.041001'
 x_serialization_backend: 'CPAN::Meta::YAML version 0.018'

bin/netdisco-sshcollector

@@ -74,6 +74,9 @@ $ENV{DBIC_TRACE} ||= $sqltrace;
 # reconfigure logging to force console output
 Dancer::Logger->init('console', $CONFIG);
 
+# silent exit unless explicitly requested
+exit(0) unless setting('use_legacy_sshcollector');
+
 if ($opensshdebug){
     $Net::OpenSSH::debug = ~0;
 }

lib/App/Netdisco.pm

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use 5.010_000;
 
-our $VERSION = '2.040007';
+our $VERSION = '2.041001';
 use App::Netdisco::Configuration;
 
 =head1 NAME
@@ -57,9 +57,9 @@ L<Docker images|https://store.docker.com/community/images/netdisco/netdisco> are
 =back
 
 We have several other pages with tips for
-L<alternate deployment scenarios|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
+L<installation tips|https://github.com/netdisco/netdisco/wiki/Install-Tips>,
 L<understanding and troubleshooting Netdisco|https://github.com/netdisco/netdisco/wiki/Troubleshooting>,
-L<tips and tricks for specific platforms|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
+L<notes for specific device vendors|https://github.com/netdisco/netdisco/wiki/Vendor-Tips>,
 and L<all the configuration options|https://github.com/netdisco/netdisco/wiki/Configuration>.
 
 You can also speak to someone in the C<#netdisco@freenode> IRC channel, or on
@@ -70,16 +70,19 @@ L<Release Notes|https://github.com/netdisco/netdisco/wiki/Release-Notes>.
 =head1 Dependencies
 
 Netdisco has several Perl library dependencies which will be automatically
-installed. However it's I<strongly> recommended that you first install
+installed. However it's required that you first install the following
-L<DBD::Pg>, L<SNMP>, and a compiler using your operating system packages.
+operating system packages:
 
 On Ubuntu/Debian:
 
- root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl build-essential
+ root:~# apt-get install libdbd-pg-perl libsnmp-perl libssl-dev libio-socket-ssl-perl curl postgresql build-essential
 
 On Fedora/Red-Hat:
 
- root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel make automake gcc
+ root:~# yum install perl-core perl-DBD-Pg net-snmp-perl net-snmp-devel openssl-devel curl postgresql-server postgresql-contrib make automake gcc
+ root:~# postgresql-setup initdb
+ root:~# systemctl start postgresql
+ root:~# systemctl enable postgresql
 
 On BSD systems please see L<our BSD tips|https://github.com/netdisco/netdisco/wiki/BSD-Install>.
 
@@ -104,9 +107,11 @@ application:
   
  postgres:~$ createdb -O netdisco netdisco
 
-The default PostgreSQL configuration isn't well tuned for modern server
+You may wish to L<amend the PostgreSQL
-hardware. We strongly recommend that you use the C<pgtune> Python program to
+configuration|https://github.com/netdisco/netdisco/wiki/Install-Tips#enable-md5-authentication-to-postgresql>
-auto-tune your C<postgresql.conf> file:
+so that local connections are working.  The default PostgreSQL configuration
+also needs tuning for modern server hardware. We recommend that you use the
+C<pgtune> Python program to auto-tune your C<postgresql.conf> file:
 
 =over 4
 

lib/App/Netdisco/Backend/Job.pm

@@ -93,7 +93,7 @@ phase.
 
 sub finalise_status {
   my $job = shift;
-  # use DDP; p $job->_statuslist;
+  # use DDP; p $job->_statuslist;
 
   # fallback
   $job->status('error');
@@ -103,7 +103,7 @@ sub finalise_status {
 
   foreach my $status (reverse @{ $job->_statuslist }) {
     next if $status->phase 
-      and $status->phase !~ m/^(?:check|early|main)$/;
+      and $status->phase !~ m/^(?:check|early|main|store|late)$/;
 
     # done() from check phase should not be the action's done()
     next if $status->phase eq 'check' and $status->is_ok;

lib/App/Netdisco/Configuration.pm

@@ -1,7 +1,7 @@
 package App::Netdisco::Configuration;
 
 use App::Netdisco::Environment;
-use App::Netdisco::Util::SNMP ();
+use App::Netdisco::Util::DeviceAuth ();
 use Dancer ':script';
 
 use Path::Class 'dir';
@@ -84,7 +84,9 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') })
   config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ];
 }
 # fix up device_auth (or create it from old snmp_auth and community settings)
-config->{'device_auth'} = [ App::Netdisco::Util::SNMP::fixup_device_auth() ];
+# also imports legacy sshcollcetor config
+config->{'device_auth'}
+  = [ App::Netdisco::Util::DeviceAuth::fixup_device_auth() ];
 
 # defaults for workers
 setting('workers')->{queue} ||= 'PostgreSQL';

lib/App/Netdisco/Transport/SNMP.pm

@@ -25,7 +25,7 @@ App::Netdisco::Transport::SNMP
 Singleton for SNMP connections. Returns cached L<SNMP::Info> instance for a
 given device IP, or else undef. All methods are class methods, for example:
 
- App::Netdisco::Transport::SNMP->reader_for( ... );
+ my $snmp = App::Netdisco::Transport::SNMP->reader_for( ... );
 
 =cut
 

lib/App/Netdisco/Transport/SSH.pm

@@ -0,0 +1,115 @@
+package App::Netdisco::Transport::SSH;
+
+use Dancer qw/:syntax :script/;
+
+use App::Netdisco::Util::Device 'get_device';
+use Module::Load ();
+use Net::OpenSSH;
+use Try::Tiny;
+
+use base 'Dancer::Object::Singleton';
+
+=head1 NAME
+
+App::Netdisco::Transport::SSH
+
+=head1 DESCRIPTION
+
+Returns an object which has an active SSH connection which can be used
+for some actions such as arpnip.
+
+ my $cli = App::Netdisco::Transport::SSH->session_for( ... );
+
+=cut
+
+__PACKAGE__->attributes(qw/ sessions /);
+
+sub init {
+  my ( $class, $self ) = @_;
+  $self->sessions( {} );
+  return $self;
+}
+
+=head1 session_for( $ip )
+
+Given an IP address, returns an object instance configured for and connected
+to that device.
+
+Returns C<undef> if the connection fails.
+
+=cut
+
+{
+  package MySession;
+  use Moo;
+
+  has 'ssh'  => ( is => 'rw' );
+  has 'auth' => ( is => 'rw' );
+  has 'host' => ( is => 'rw' );
+  has 'platform' => ( is => 'rw' );
+
+  sub arpnip {
+    my $self = shift;
+    $self->platform->arpnip(@_, $self->host, $self->ssh, $self->auth);
+  }
+}
+
+sub session_for {
+  my ($class, $ip) = @_;
+
+  my $device = get_device($ip) or return undef;
+  my $sessions = $class->instance->sessions or return undef;
+
+  return $sessions->{$device->ip} if exists $sessions->{$device->ip};
+  debug sprintf 'cli session cache warm: [%s]', $device->ip;
+
+  my $auth = (setting('device_auth') || []);
+  if (1 != scalar @$auth) {
+    error sprintf " [%s] require only one matching auth stanza", $device->ip;
+    return undef;
+  }
+  $auth = $auth->[0];
+
+  my @master_opts = qw(-o BatchMode=no);
+  push(@master_opts, @{$auth->{ssh_master_opts}})
+    if $auth->{ssh_master_opts};
+
+  $Net::OpenSSH::debug = $ENV{SSH_TRACE};
+  my $ssh = Net::OpenSSH->new(
+    $device->ip,
+    user => $auth->{username},
+    password => $auth->{password},
+    timeout => 30,
+    async => 0,
+    default_stderr_file => '/dev/null',
+    master_opts => \@master_opts
+  );
+
+  if ($ssh->error) {
+    error sprintf " [%s] ssh connection error [%s]", $device->ip, $ssh->error;
+    return undef;
+  }
+  elsif (! $ssh) {
+    error sprintf " [%s] Net::OpenSSH instantiation error", $device->ip;
+    return undef;
+  }
+
+  my $platform = "App::Netdisco::SSHCollector::Platform::" . $auth->{platform};
+  my $happy = false;
+  try {
+    Module::Load::load $platform;
+    $happy = true;
+  } catch { error $_ };
+  return unless $happy;
+
+  my $sess = MySession->new(
+    ssh  => $ssh,
+    auth => $auth,
+    host => $device->ip,
+    platform => $platform->new(),
+  );
+
+  return ($sessions->{$device->ip} = $sess);
+}
+
+true;

lib/App/Netdisco/Util/DeviceAuth.pm

@@ -0,0 +1,179 @@
+package App::Netdisco::Util::DeviceAuth;
+
+use Dancer qw/:syntax :script/;
+use App::Netdisco::Util::DNS 'hostname_from_ip';
+
+use Try::Tiny;
+
+use base 'Exporter';
+our @EXPORT = ();
+our @EXPORT_OK = qw/
+  fixup_device_auth get_external_credentials
+/;
+our %EXPORT_TAGS = (all => \@EXPORT_OK);
+
+=head1 NAME
+
+App::Netdisco::Util::DeviceAuth
+
+=head1 DESCRIPTION
+
+Helper functions for device authentication.
+
+There are no default exports, however the C<:all> tag will export all
+subroutines.
+
+=head1 EXPORT_OK
+
+=head2 fixup_device_auth
+
+Rebuilds the C<device_auth> config with missing defaults and other fixups for
+config changes over time. Returns a list which can replace C<device_auth>.
+
+=cut
+
+sub fixup_device_auth {
+  my $config = (setting('snmp_auth') || setting('device_auth'));
+  my @new_stanzas = ();
+
+  # new style snmp config
+  foreach my $stanza (@$config) {
+    # user tagged
+    my $tag = '';
+    if (1 == scalar keys %$stanza) {
+      $tag = (keys %$stanza)[0];
+      $stanza = $stanza->{$tag};
+
+      # corner case: untagged lone community
+      if ($tag eq 'community') {
+          $tag = $stanza;
+          $stanza = {community => $tag};
+      }
+    }
+
+    # defaults
+    $stanza->{tag} ||= $tag;
+    $stanza->{read} = 1 if !exists $stanza->{read};
+    $stanza->{no}   ||= [];
+    $stanza->{only} ||= ['any'];
+
+    die "error: config: snmpv2 community in device_auth must be single item, not list\n"
+      if ref $stanza->{community};
+
+    die "error: config: stanza in device_auth must have a tag\n"
+      if not $stanza->{tag} and exists $stanza->{user};
+
+    push @new_stanzas, $stanza;
+  }
+
+  # import legacy sshcollector configuration
+  my $sshcollector = (setting('sshcollector') || []);
+  foreach my $stanza (@$sshcollector) {
+    # defaults
+    $stanza->{driver} = 'cli';
+    $stanza->{read} = 1;
+    $stanza->{no}   ||= [];
+
+    # fixups
+    $stanza->{only} ||= [ scalar delete $stanza->{ip} ||
+                          scalar delete $stanza->{hostname} ];
+    $stanza->{username} = scalar delete $stanza->{user};
+
+    push @new_stanzas, $stanza;
+  }
+
+  # legacy config 
+  # note: read strings tried before write
+  # note: read-write is no longer used for read operations
+
+  push @new_stanzas, map {{
+    read => 1, write => 0,
+    no => [], only => ['any'],
+    community => $_,
+  }} @{setting('community') || []};
+
+  push @new_stanzas, map {{
+    write => 1, read => 0,
+    no => [], only => ['any'],
+    community => $_,
+  }} @{setting('community_rw') || []};
+
+  foreach my $stanza (@new_stanzas) {
+    $stanza->{driver} ||= 'snmp'
+      if exists $stanza->{community}
+         or exists $stanza->{user};
+  }
+
+  return @new_stanzas;
+}
+
+=head2 get_external_credentials( $device, $mode )
+
+Runs a command to gather SNMP credentials or a C<device_auth> stanza.
+
+Mode can be C<read> or C<write> and defaults to 'read'.
+
+=cut
+
+sub get_external_credentials {
+  my ($device, $mode) = @_;
+  my $cmd = (setting('get_credentials') || setting('get_community'));
+  my $ip = $device->ip;
+  my $host = ($device->dns || hostname_from_ip($ip) || $ip);
+  $mode ||= 'read';
+
+  if (defined $cmd and length $cmd) {
+      # replace variables
+      $cmd =~ s/\%MODE\%/$mode/egi;
+      $cmd =~ s/\%HOST\%/$host/egi;
+      $cmd =~ s/\%IP\%/$ip/egi;
+
+      my $result = `$cmd`; # BACKTICKS
+      return () unless defined $result and length $result;
+
+      my @lines = split (m/\n/, $result);
+      foreach my $line (@lines) {
+          if ($line =~ m/^community\s*=\s*(.*)\s*$/i) {
+              if (length $1 and $mode eq 'read') {
+                  debug sprintf '[%s] external read credentials added',
+                    $device->ip;
+
+                  return map {{
+                    read => 1,
+                    only => [$device->ip],
+                    community => $_,
+                  }} split(m/\s*,\s*/,$1);
+              }
+          }
+          elsif ($line =~ m/^setCommunity\s*=\s*(.*)\s*$/i) {
+              if (length $1 and $mode eq 'write') {
+                  debug sprintf '[%s] external write credentials added',
+                    $device->ip;
+
+                  return map {{
+                    write => 1,
+                    only => [$device->ip],
+                    community => $_,
+                  }} split(m/\s*,\s*/,$1);
+              }
+          }
+          else {
+            my $stanza = undef;
+            try {
+              $stanza = from_json( $line );
+              debug sprintf '[%s] external credentials stanza added',
+                $device->ip;
+            }
+            catch {
+              info sprintf '[%s] error! failed to parse external credentials stanza',
+                $device->ip;
+            };
+            return $stanza if ref $stanza;
+          }
+      }
+  }
+
+  return ();
+}
+
+true;

lib/App/Netdisco/Util/Port.pm

@@ -219,7 +219,7 @@ Returns true if the C<$port> L<DBIx::Class> object has a phone connected.
 =cut
 
 sub port_has_phone {
-  return (shift)->with_properties->remote_is_phone;
+  return (shift)->properties->remote_is_phone;
 }
 
 1;

lib/App/Netdisco/Util/SNMP.pm

@@ -1,14 +1,11 @@
 package App::Netdisco::Util::SNMP;
 
 use Dancer qw/:syntax :script/;
-use App::Netdisco::Util::DNS 'hostname_from_ip';
+use App::Netdisco::Util::DeviceAuth 'get_external_credentials';
-use App::Netdisco::Util::Permission ':all';
 
 use base 'Exporter';
 our @EXPORT = ();
-our @EXPORT_OK = qw/
+our @EXPORT_OK = qw/ get_communities snmp_comm_reindex /;
-  fixup_device_auth get_communities snmp_comm_reindex
-/;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
 =head1 NAME
@@ -24,72 +21,6 @@ subroutines.
 
 =head1 EXPORT_OK
 
-=head2 fixup_device_auth
-
-Rebuilds the C<device_auth> config with missing defaults and other fixups for
-config changes over time. Returns a list which can replace C<device_auth>.
-
-=cut
-
-sub fixup_device_auth {
-  my $config = (setting('snmp_auth') || setting('device_auth'));
-  my @new_stanzas = ();
-
-  # new style snmp config
-  foreach my $stanza (@$config) {
-    # user tagged
-    my $tag = '';
-    if (1 == scalar keys %$stanza) {
-      $tag = (keys %$stanza)[0];
-      $stanza = $stanza->{$tag};
-
-      # corner case: untagged lone community
-      if ($tag eq 'community') {
-          $tag = $stanza;
-          $stanza = {community => $tag};
-      }
-    }
-
-    # defaults
-    $stanza->{tag} ||= $tag;
-    $stanza->{read} = 1 if !exists $stanza->{read};
-    $stanza->{no}   ||= [];
-    $stanza->{only} ||= ['any'];
-
-    die "error: config: snmpv2 community in device_auth must be single item, not list\n"
-      if ref $stanza->{community};
-
-    die "error: config: stanza in device_auth must have a tag\n"
-      if not $stanza->{tag} and exists $stanza->{user};
-
-    push @new_stanzas, $stanza
-  }
-
-  # legacy config 
-  # note: read strings tried before write
-  # note: read-write is no longer used for read operations
-
-  push @new_stanzas, map {{
-    read => 1, write => 0,
-    no => [], only => ['any'],
-    community => $_,
-  }} @{setting('community') || []};
-
-  push @new_stanzas, map {{
-    write => 1, read => 0,
-    no => [], only => ['any'],
-    community => $_,
-  }} @{setting('community_rw') || []};
-
-  foreach my $stanza (@new_stanzas) {
-    $stanza->{driver} ||= 'snmp'
-      if exists $stanza->{community}
-         or exists $stanza->{user};
-  }
-
-  return @new_stanzas;
-}
-
 =head2 get_communities( $device, $mode )
 
 Takes the current C<device_auth> setting and pushes onto the front of the list
@@ -106,8 +37,7 @@ sub get_communities {
   my @communities = ();
 
   # first of all, use external command if configured
-  push @communities, _get_external_community($device, $mode)
+  push @communities, get_external_credentials($device, $mode);
-    if setting('get_community') and length setting('get_community');
 
   # last known-good by tag
   my $tag_name = 'snmp_auth_tag_'. $mode;
@@ -145,46 +75,6 @@ sub get_communities {
   return ( @communities, @$config );
 }
 
-sub _get_external_community {
-  my ($device, $mode) = @_;
-  my $cmd = setting('get_community');
-  my $ip = $device->ip;
-  my $host = ($device->dns || hostname_from_ip($ip) || $ip);
-
-  if (defined $cmd and length $cmd) {
-      # replace variables
-      $cmd =~ s/\%HOST\%/$host/egi;
-      $cmd =~ s/\%IP\%/$ip/egi;
-
-      my $result = `$cmd`; # BACKTICKS
-      return () unless defined $result and length $result;
-
-      my @lines = split (m/\n/, $result);
-      foreach my $line (@lines) {
-          if ($line =~ m/^community\s*=\s*(.*)\s*$/i) {
-              if (length $1 and $mode eq 'read') {
-                  return map {{
-                    read => 1,
-                    only => [$device->ip],
-                    community => $_,
-                  }} split(m/\s*,\s*/,$1);
-              }
-          }
-          elsif ($line =~ m/^setCommunity\s*=\s*(.*)\s*$/i) {
-              if (length $1 and $mode eq 'write') {
-                  return map {{
-                    write => 1,
-                    only => [$device->ip],
-                    community => $_,
-                  }} split(m/\s*,\s*/,$1);
-              }
-          }
-      }
-  }
-
-  return ();
-}
-
 =head2 snmp_comm_reindex( $snmp, $device, $vlan )
 
 Takes an established L<SNMP::Info> instance and makes a fresh connection using
@@ -210,11 +100,19 @@ sub snmp_comm_reindex {
       }
       $prefix ||= 'vlan-';
 
-      debug
+      if ($vlan =~ /^[0-9]+$/i && $vlan) { 
-        sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
+        debug sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)',
         $device->ip, $prefix, $vlan, $ver, $snmp->class;
-      $vlan ? $snmp->update(Context => ($prefix . $vlan))
+        $snmp->update(Context => ($prefix . $vlan));
-            : $snmp->update(Context => '');
+      } elsif ($vlan =~ /^[a-z0-9]+$/i && $vlan) { 
+        debug sprintf '[%s] reindexing to "%s" (ver: %s, class: %s)',
+          $device->ip, $vlan, $ver, $snmp->class;
+        $snmp->update(Context => ($vlan));
+      } else { 
+        debug sprintf '[%s] reindexing without context (ver: %s, class: %s)',
+          $device->ip, $ver, $snmp->class;
+        $snmp->update(Context => ''); 
+      }
   }
   else {
       my $comm = $snmp->snmp_comm;

lib/App/Netdisco/Web.pm

@@ -66,9 +66,13 @@ if (setting('template_paths') and ref [] eq ref setting('template_paths')) {
 
 # load cookie key from database
 setting('session_cookie_key' => undef);
-my $sessions = schema('netdisco')->resultset('Session');
+setting('session_cookie_key' => 'this_is_for_testing_only')
-my $skey = $sessions->find({id => 'dancer_session_cookie_key'});
+  if $ENV{HARNESS_ACTIVE};
-setting('session_cookie_key' => $skey->get_column('a_session')) if $skey;
+eval {
+  my $sessions = schema('netdisco')->resultset('Session');
+  my $skey = $sessions->find({id => 'dancer_session_cookie_key'});
+  setting('session_cookie_key' => $skey->get_column('a_session')) if $skey;
+};
 Dancer::Session::Cookie::init(session);
 
 # workaround for https://github.com/PerlDancer/Dancer/issues/935

lib/App/Netdisco/Worker/Loader.pm

@@ -12,7 +12,9 @@ use namespace::clean;
 has [qw/workers_check
         workers_early
         workers_main
-        workers_user/] => ( is => 'rw' );
+        workers_user
+        workers_store
+        workers_late/] => ( is => 'rw' );
 
 sub load_workers {
   my $self = shift;
@@ -37,7 +39,7 @@ sub load_workers {
   my $workers = vars->{'workers'}->{$action} || {};
   #use DDP; p vars->{'workers'};
 
-  foreach my $phase (qw/check early main user/) {
+  foreach my $phase (qw/check early main user store late/) {
     my $pname = "workers_${phase}";
     my @wset = ();
 

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -1,15 +1,43 @@
 package App::Netdisco::Worker::Plugin::Arpnip::Nodes;
 
 use Dancer ':syntax';
+use Dancer::Plugin::DBIC 'schema';
+
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
+use App::Netdisco::Transport::SSH ();
 use App::Netdisco::Transport::SNMP ();
+
 use App::Netdisco::Util::Node qw/check_mac store_arp/;
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
-use Dancer::Plugin::DBIC 'schema';
+
+use NetAddr::IP::Lite ':lower';
 use Time::HiRes 'gettimeofday';
 
+register_worker({ phase => 'store' }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  # would be possible just to use now() on updated records, but by using this
+  # same value for them all, we _can_ if we want add a job at the end to
+  # select and do something with the updated set (no reason to yet, though)
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  # update node_ip with ARP and Neighbor Cache entries
+
+  store_arp(\%$_, $now) for @{ vars->{'v4arps'} };
+  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
+    $device->ip, scalar @{ vars->{'v4arps'} };
+
+  store_arp(\%$_, $now) for @{ vars->{'v6arps'} };
+  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
+    $device->ip, scalar @{ vars->{'v6arps'} };
+
+  $device->update({last_arpnip => \$now});
+  return Status->done("Ended arpnip for $device");
+});
+
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my ($job, $workerconf) = @_;
 
@@ -17,31 +45,19 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
     or return Status->defer("arpnip failed: could not SNMP connect to $device");
 
-  # get v4 arp table
+  # cache v4 arp table
-  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
+  push @{ vars->{'v4arps'} },
-  # get v6 neighbor cache
+    @{ get_arps_snmp($device, $snmp->at_paddr, $snmp->at_netaddr) };
-  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
 
-  # would be possible just to use now() on updated records, but by using this
+  # cache v6 neighbor cache
-  # same value for them all, we _can_ if we want add a job at the end to
+  push @{ vars->{'v6arps'} },
-  # select and do something with the updated set (no reason to yet, though)
+    @{get_arps_snmp($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr) };
-  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
 
-  # update node_ip with ARP and Neighbor Cache entries
+  return Status->info("Gathered arp caches from $device");
-  store_arp(\%$_, $now) for @$v4;
-  debug sprintf ' [%s] arpnip - processed %s ARP Cache entries',
-    $device->ip, scalar @$v4;
-
-  store_arp(\%$_, $now) for @$v6;
-  debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
-    $device->ip, scalar @$v6;
-
-  $device->update({last_arpnip => \$now});
-  return Status->done("Ended arpnip for $device");
 });
 
 # get an arp table (v4 or v6)
-sub get_arps {
+sub get_arps_snmp {
   my ($device, $paddr, $netaddr) = @_;
   my @arps = ();
 
@@ -63,4 +79,46 @@ sub get_arps {
   return $resolved_ips;
 }
 
+register_worker({ phase => 'main', driver => 'cli' }, sub {
+    my ($job, $workerconf) = @_;
+
+    my $device = $job->device;
+    my $cli = App::Netdisco::Transport::SSH->session_for($device)
+      or return Status->defer("arpnip failed: could not SSH connect to $device");
+
+    # should be both v4 and v6
+    my @arps = @{ get_arps_cli($device, [$cli->arpnip]) };
+
+    # cache v4 arp table
+    push @{ vars->{'v4arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits ==  32 } @arps;
+
+    # cache v6 neighbor cache
+    push @{ vars->{'v6arps'} },
+      grep { NetAddr::IP::Lite->new($_->{ip})->bits == 128 } @arps;
+
+    return Status->info("Gathered arp caches from $device");
+});
+
+sub get_arps_cli {
+  my ($device, $entries) = @_;
+  my @arps = ();
+  $entries ||= [];
+
+  foreach my $entry (@$entries) {
+    next unless check_mac($entry->{mac}, $device);
+    push @arps, {
+        node => $entry->{mac},
+        ip   => $entry->{ip},
+        dns  => $entry->{dns},
+    };
+  }
+
+  debug sprintf ' resolving %d ARP entries with max %d outstanding requests',
+    scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+  my $resolved_ips = hostnames_resolve_async(\@arps);
+
+  return $resolved_ips;
+}
+
 true;

lib/App/Netdisco/Worker/Plugin/DumpConfig.pm

@@ -11,7 +11,8 @@ register_worker({ phase => 'main' }, sub {
   my $extra = $job->extra;
 
   my $config = config();
-  p ($extra ? $config->{$extra} : $config);
+  my $dump = ($extra ? $config->{$extra} : $config);
+  p $dump;
   return Status->done('Dumped config');
 });
 

lib/App/Netdisco/Worker/Runner.pm

@@ -69,7 +69,7 @@ sub run {
 
     # run other phases
     if ($job->check_passed) {
-      $self->run_workers("workers_${_}") for qw/early main user/;
+      $self->run_workers("workers_${_}") for qw/early main user store late/;
     }
   };
 

share/config.yml

@@ -216,7 +216,8 @@ device_identity: []
 community: []
 community_rw: []
 device_auth: []
-get_community: ""
+use_legacy_sshcollector: false
+get_credentials: ""
 bulkwalk_off: false
 bulkwalk_no: []
 bulkwalk_repeaters: 20
@@ -414,7 +415,6 @@ worker_plugins:
   - 'Vlan::Core'
 
 extra_worker_plugins: []
-#  - Discover::ConfigBackup::CLI
 
 driver_priority:
   restconf: 500

xt/00-compile.t

@@ -0,0 +1,30 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+BEGIN {
+  use FindBin;
+  FindBin::again();
+
+  use Path::Class;
+
+  # stuff useful locations into @INC and $PATH
+  unshift @INC,
+    dir($FindBin::RealBin)->parent->subdir('lib')->stringify,
+    dir($FindBin::RealBin, 'lib')->stringify;
+}
+
+# for netdisco app config
+use App::Netdisco;
+use Test::Compile;
+
+my $test = Test::Compile->new();
+
+my @plfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pl_files();
+my @pmfiles = grep {$_ !~ m/(?:graph)/i} $test->all_pm_files();
+
+$test->ok($test->pl_file_compiles($_), "$_ compiles") for @plfiles;
+$test->ok($test->pm_file_compiles($_), "$_ compiles") for @pmfiles;
+
+$test->done_testing();