implement ignore_layers, force_macsuck, force_arpnip config settings (#1002)

* implementation of ignore_layers, force_macsuck, force_arpnip and macwalk

* use new WalkJobs view to get devices needing macsuck

* also new query for discoverall, arpwalk, nbtwalk

* faux record has a last_defer stamp so we can see when the backend started

* fix typo

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -65,7 +65,6 @@ register_worker({ phase => 'store' }, sub {
     $device->ip, $v6;
 
   $device->update({last_arpnip => \$now});
-  $device->update({layers => \[q{overlay(layers placing '1' from 6 for 1)}]});
 
   my $status = $job->best_status;
   return Status->$status("Ended arpnip for $device");

lib/App/Netdisco/Worker/Plugin/Arpwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('arpnip');
-  my @devices = schema('netdisco')->resultset('Device')
-    ->has_layer('3')->get_column('ip')->all;
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
+    ->search(undef,{ bind => [
+      'arpnip', 'arpnip',
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'arpnip',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued arpnip job for all devices');

lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm

@@ -6,6 +6,7 @@ use App::Netdisco::Transport::SNMP;
 use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::Util::Device qw/get_device is_discoverable/;
+use App::Netdisco::Util::Permission 'check_acl_no';
 use App::Netdisco::JobQueue 'jq_insert';
 
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
@@ -13,7 +14,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   return unless setting('discover_routed_neighbors');
 
   my $device = $job->device;
-  return unless $device->in_storage and $device->has_layer(3);
+  return unless $device->in_storage and ($device->has_layer(3)
+                                         or check_acl_no($device, 'force_macsuck')
+                                         or check_acl_no($device, 'ignore_layers'));
+
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
     or return Status->defer("discover failed: could not SNMP connect to $device");
 

lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm

@@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::JobQueue 'jq_insert';
-use Dancer::Plugin::DBIC 'schema';
+use App::Netdisco::Util::Permission 'check_acl_no';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
@@ -15,25 +15,29 @@ register_worker({ phase => 'main' }, sub {
   # arpniped/macsucked, queue those jobs now
   return unless $device->in_storage and $job->subaction eq 'with-nodes';
 
-  if (!defined $device->last_macsuck and $device->has_layer(2)) {
+  if (!defined $device->last_macsuck and ($device->has_layer(2)
+                                          or check_acl_no($device, 'force_macsuck')
+                                          or check_acl_no($device, 'ignore_layers'))) {
     jq_insert({
       device => $device->ip,
       action => 'macsuck',
       username => $job->username,
       userip => $job->userip,
     });
+    debug sprintf ' [%s] queued macsuck', $device;
   }
 
-  if (!defined $device->last_arpnip and $device->has_layer(3)) {
+  if (!defined $device->last_arpnip and ($device->has_layer(3)
+                                         or check_acl_no($device, 'force_arpnip')
+                                         or check_acl_no($device, 'ignore_layers'))) {
     jq_insert({
       device => $device->ip,
       action => 'arpnip',
       username => $job->username,
       userip => $job->userip,
     });
+    debug sprintf ' [%s] queued arpnip', $device;
   }
-
-  return Status->info("Queued macsuck and arpnip for $device.");
 });
 
 true;

lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm

@@ -4,17 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('discover');
-  my @devices = schema('netdisco')->resultset('Device')->search({
-    -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
-  })->get_column('ip')->all;
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
+    ->search(undef,{ bind => [
+      'discover', 'discover',
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -22,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'discover',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued discover job for all devices');

lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm

@@ -207,7 +207,6 @@ register_worker({ phase => 'store',
     $device->ip, $archived;
 
   $device->update({last_macsuck => \$now});
-  $device->update({layers => \[q{overlay(layers placing '1' from 7 for 1)}]});
 
   my $status = $job->best_status;
   return Status->$status("Ended macsuck for $device");

lib/App/Netdisco/Worker/Plugin/Macwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('macsuck');
-  my @devices = schema('netdisco')->resultset('Device')
-    ->has_layer('2')->get_column('ip')->all;
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
+    ->search(undef,{ bind => [
+      'macsuck', 'macsuck',
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'macsuck',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued macsuck job for all devices');

lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('nbtstat');
-  my @devices = schema('netdisco')->resultset('Device')
-    ->has_layer('2')->get_column('ip')->all;
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
+    ->search(undef,{ bind => [
+      'macsuck', 'macsuck',
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'nbtstat',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued nbtstat job for all devices');