Merge remote-tracking branch 'origin/master'

lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm

@@ -0,0 +1,43 @@
+package App::Netdisco::DB::Result::Virtual::WalkJobs;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
+
+__PACKAGE__->table('walk_jobs');
+__PACKAGE__->result_source_instance->is_virtual(1);
+__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
+    SELECT ip
+    FROM device
+
+    LEFT OUTER JOIN admin ON (device.ip = admin.device
+                              AND admin.status = 'queued'
+                              AND admin.action = ?)
+
+    FULL OUTER JOIN device_skip ON (device_skip.device = device.ip
+                                    AND (device_skip.actionset @> string_to_array(?, '')
+                                         OR (device_skip.deferrals >= ?
+                                             AND device_skip.last_defer > (LOCALTIMESTAMP - ? ::interval))))
+
+    WHERE admin.device IS NULL
+      AND device.ip IS NOT NULL
+      AND (device.vendor IS NULL OR device.vendor != 'netdisco')
+
+    GROUP BY device.ip
+    HAVING count(device_skip.backend) < (SELECT count(distinct(backend)) FROM device_skip)
+
+    ORDER BY device.ip ASC
+ENDSQL
+);
+
+__PACKAGE__->add_columns(
+  "ip",
+  { data_type => "inet", is_nullable => 0 },
+);
+
+__PACKAGE__->set_primary_key("ip");
+
+1;

lib/App/Netdisco/JobQueue/PostgreSQL.pm

@@ -72,6 +72,8 @@ sub jq_warm_thrusters {
       backend => setting('workers')->{'BACKEND'},
     }, { for => 'update' }, )->update({ actionset => [] });
 
+    # on backend restart, allow one retry of all devices which have
+    # reached max retry (max_deferrals)
     my $deferrals = setting('workers')->{'max_deferrals'} - 1;
     $rs->search({
       backend => setting('workers')->{'BACKEND'},
@@ -80,7 +82,7 @@ sub jq_warm_thrusters {
 
     $rs->search({
       backend => setting('workers')->{'BACKEND'},
-      actionset => { -value => [] },
+      actionset => { -value => [] }, # special syntax for matching empty ARRAY
       deferrals => 0,
     })->delete;
 
@@ -89,19 +91,13 @@ sub jq_warm_thrusters {
       device  => $_,
       actionset => $actionset{$_},
     }, { key => 'primary' }) for keys %actionset;
-  });
 
-  # fix up the pseudo devices which need layer 3
+    # add one faux record to allow *walk actions to see there is a backend running
-  # TODO remove this after next release
+    $rs->update_or_create({
-  schema(vars->{'tenant'})->txn_do(sub {
+      backend => setting('workers')->{'BACKEND'},
-    my @hosts = grep { defined }
+      device  => '255.255.255.255',
-                map  { schema(vars->{'tenant'})->resultset('Device')->search_for_device($_->{only}) }
+      last_defer => \'LOCALTIMESTAMP',
-                grep { exists $_->{only} and ref '' eq ref $_->{only} }
+    }, { key => 'primary' });
-                grep { exists $_->{driver} and $_->{driver} eq 'cli' }
-                    @{ setting('device_auth') };
-
-    $_->update({ layers => \[q{overlay(layers placing '1' from 6 for 1)}] })
-      for @hosts;
   });
 }
 
@@ -166,7 +162,7 @@ sub jq_getsome {
         },{
           job => $job->id,
           -exists => $jobs->search({
-	    job => { '>' => $job->id },
+            job => { '>' => $job->id },
             status => { -like => 'queued-%' },
             started => \[q/> (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')],
             %job_properties,

lib/App/Netdisco/Util/Device.pm

@@ -230,7 +230,8 @@ the local configuration to arpnip the device.
 The configuration items C<arpnip_no> and C<arpnip_only> are checked
 against the given IP.
 
-Also checks if the device reports layer 3 capability.
+Also checks if the device reports layer 3 capability, or matches
+C<force_arpnip> or C<ignore_layers>.
 
 Returns false if the host is not permitted to arpnip the target device.
 
@@ -241,7 +242,9 @@ sub is_arpnipable {
   my $device = get_device($ip) or return 0;
 
   return _bail_msg("is_arpnipable: $device has no layer 3 capability")
-    if ($device->in_storage() and not $device->has_layer(3));
+    if ($device->in_storage() and not ($device->has_layer(3)
+                                       or check_acl_no($device, 'force_arpnip')
+                                       or check_acl_no($device, 'ignore_layers')));
 
   return _bail_msg("is_arpnipable: $device matched arpnip_no")
     if check_acl_no($device, 'arpnip_no');
@@ -283,7 +286,8 @@ the local configuration to macsuck the device.
 The configuration items C<macsuck_no> and C<macsuck_only> are checked
 against the given IP.
 
-Also checks if the device reports layer 2 capability.
+Also checks if the device reports layer 2 capability, or matches
+C<force_macsuck> or C<ignore_layers>.
 
 Returns false if the host is not permitted to macsuck the target device.
 
@@ -294,7 +298,9 @@ sub is_macsuckable {
   my $device = get_device($ip) or return 0;
 
   return _bail_msg("is_macsuckable: $device has no layer 2 capability")
-    if ($device->in_storage() and not $device->has_layer(2));
+    if ($device->in_storage() and not ($device->has_layer(2)
+                                       or check_acl_no($device, 'force_macsuck')
+                                       or check_acl_no($device, 'ignore_layers')));
 
   return _bail_msg("is_macsuckable: $device matched macsuck_no")
     if check_acl_no($device, 'macsuck_no');

lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm

@@ -65,7 +65,6 @@ register_worker({ phase => 'store' }, sub {
     $device->ip, $v6;
 
   $device->update({last_arpnip => \$now});
-  $device->update({layers => \[q{overlay(layers placing '1' from 6 for 1)}]});
 
   my $status = $job->best_status;
   return Status->$status("Ended arpnip for $device");

lib/App/Netdisco/Worker/Plugin/Arpwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('arpnip');
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
-  my @devices = schema('netdisco')->resultset('Device')
+    ->search(undef,{ bind => [
-    ->has_layer('3')->get_column('ip')->all;
+      'arpnip', 'arpnip',
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'arpnip',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued arpnip job for all devices');

lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm

@@ -6,6 +6,7 @@ use App::Netdisco::Transport::SNMP;
 use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::Util::Device qw/get_device is_discoverable/;
+use App::Netdisco::Util::Permission 'check_acl_no';
 use App::Netdisco::JobQueue 'jq_insert';
 
 register_worker({ phase => 'main', driver => 'snmp' }, sub {
@@ -13,7 +14,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   return unless setting('discover_routed_neighbors');
 
   my $device = $job->device;
-  return unless $device->in_storage and $device->has_layer(3);
+  return unless $device->in_storage and ($device->has_layer(3)
+                                         or check_acl_no($device, 'force_macsuck')
+                                         or check_acl_no($device, 'ignore_layers'));
+
   my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
     or return Status->defer("discover failed: could not SNMP connect to $device");
 

lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm

@@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::JobQueue 'jq_insert';
-use Dancer::Plugin::DBIC 'schema';
+use App::Netdisco::Util::Permission 'check_acl_no';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
@@ -15,25 +15,29 @@ register_worker({ phase => 'main' }, sub {
   # arpniped/macsucked, queue those jobs now
   return unless $device->in_storage and $job->subaction eq 'with-nodes';
 
-  if (!defined $device->last_macsuck and $device->has_layer(2)) {
+  if (!defined $device->last_macsuck and ($device->has_layer(2)
+                                          or check_acl_no($device, 'force_macsuck')
+                                          or check_acl_no($device, 'ignore_layers'))) {
     jq_insert({
       device => $device->ip,
       action => 'macsuck',
       username => $job->username,
       userip => $job->userip,
     });
+    debug sprintf ' [%s] queued macsuck', $device;
   }
 
-  if (!defined $device->last_arpnip and $device->has_layer(3)) {
+  if (!defined $device->last_arpnip and ($device->has_layer(3)
+                                         or check_acl_no($device, 'force_arpnip')
+                                         or check_acl_no($device, 'ignore_layers'))) {
     jq_insert({
       device => $device->ip,
       action => 'arpnip',
       username => $job->username,
       userip => $job->userip,
     });
+    debug sprintf ' [%s] queued arpnip', $device;
   }
-
-  return Status->info("Queued macsuck and arpnip for $device.");
 });
 
 true;

lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm

@@ -4,17 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('discover');
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
-  my @devices = schema('netdisco')->resultset('Device')->search({
+    ->search(undef,{ bind => [
-    -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
+      'discover', 'discover',
-  })->get_column('ip')->all;
+      setting('workers')->{'max_deferrals'},
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -22,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'discover',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued discover job for all devices');

lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm

@@ -207,7 +207,6 @@ register_worker({ phase => 'store',
     $device->ip, $archived;
 
   $device->update({last_macsuck => \$now});
-  $device->update({layers => \[q{overlay(layers placing '1' from 7 for 1)}]});
 
   my $status = $job->best_status;
   return Status->$status("Ended macsuck for $device");

lib/App/Netdisco/Worker/Plugin/Macwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('macsuck');
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
-  my @devices = schema('netdisco')->resultset('Device')
+    ->search(undef,{ bind => [
-    ->has_layer('2')->get_column('ip')->all;
+      'macsuck', 'macsuck',
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'macsuck',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued macsuck job for all devices');

lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm

@@ -4,16 +4,18 @@ use Dancer ':syntax';
 use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 
-  my %queued = map {$_ => 1} jq_queued('nbtstat');
+  my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
-  my @devices = schema('netdisco')->resultset('Device')
+    ->search(undef,{ bind => [
-    ->has_layer('2')->get_column('ip')->all;
+      'macsuck', 'macsuck',
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+      setting('workers')->{'max_deferrals'},
+      setting('workers')->{'retry_after'},
+    ]})->get_column('ip')->all;
 
   jq_insert([
     map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
       action => 'nbtstat',
       username => $job->username,
       userip => $job->userip,
-    }} (@filtered_devices)
+    }} (@walk)
   ]);
 
   return Status->done('Queued nbtstat job for all devices');

share/config.yml

@@ -351,6 +351,8 @@ discover_waps: true
 discover_phones: false
 discover_routed_neighbors: true
 discover_min_age: 0
+ignore_layers: []
+force_macsuck: []
 macsuck_no: []
 macsuck_only: []
 macsuck_all_vlans: false
@@ -373,6 +375,7 @@ macsuck_min_age: 0
 snmpforce_v1: []
 snmpforce_v2: []
 snmpforce_v3: []
+force_arpnip: []
 arpnip_no: []
 arpnip_only: []
 arpnip_min_age: 0