#1084 move slow skiplist build to the first job on running backend

bin/netdisco-backend-fg

@@ -35,7 +35,8 @@ use Role::Tiny::With;
 
 use MCE::Signal '-setpgrp';
 use MCE::Flow Sereal => 1;
-use MCE::Queue;
+use MCE::Queue porder => $MCE::Queue::HIGHEST,
+               type   => $MCE::Queue::FIFO;
 
 # set temporary MCE files' location in home directory
 my $home = ($ENV{NETDISCO_HOME} || $ENV{HOME});

lib/App/Netdisco/Backend/Role/Manager.pm

@@ -5,6 +5,7 @@ use Dancer qw/:moose :syntax :script/;
 use List::Util 'sum';
 use App::Netdisco::Util::MCE;
 
+use App::Netdisco::Backend::Job;
 use App::Netdisco::JobQueue
   qw/jq_locked jq_getsome jq_lock jq_warm_thrusters/;
 
@@ -21,9 +22,13 @@ sub worker_begin {
   debug "entering Manager ($wid) worker_begin()";
 
   # job queue initialisation
-  debug "mgr ($wid): building acl hints (please be patient...)";
+  # the expensive parts of this were moved to primeskiplist job
   jq_warm_thrusters;
 
+  # queue a job to rebuild the device action skip list
+  $self->{queue}->enqueuep(200,
+    App::Netdisco::Backend::Job->new({ job => 0, action => 'primeskiplist' }));
+
   # requeue jobs locally
   debug "mgr ($wid): searching for jobs booked to this processing node";
   my @jobs = jq_locked;
@@ -60,7 +65,7 @@ sub worker_body {
       my %seen_job = ();
 
       $num_slots = parse_max_workers( setting('workers')->{tasks} )
-                     - $self->{queue}->pending();
+                      - $self->{queue}->pending();
       debug "mgr ($wid): getting potential jobs for $num_slots workers";
 
       foreach my $job ( jq_getsome($num_slots) ) {

lib/App/Netdisco/JobQueue/PostgreSQL.pm

@@ -3,11 +3,9 @@ package App::Netdisco::JobQueue::PostgreSQL;
 use Dancer qw/:moose :syntax :script/;
 use Dancer::Plugin::DBIC 'schema';
 
-use App::Netdisco::Util::Device
+use App::Netdisco::Util::Device 'get_denied_actions';
-  qw/get_device is_discoverable is_macsuckable is_arpnipable/;
 use App::Netdisco::Backend::Job;
 
-use Module::Load ();
 use JSON::PP ();
 use Try::Tiny;
 
@@ -28,45 +26,8 @@ our @EXPORT_OK = qw/
 /;
 our %EXPORT_TAGS = ( all => \@EXPORT_OK );
 
-# given a device, tests if any of the primary acls applies
-# returns a list of job actions to be denied/skipped on this host.
-sub _get_denied_actions {
-  my $device = shift;
-  my @badactions = ();
-  return @badactions unless $device;
-  $device = get_device($device); # might be no-op but is done in is_* anyway
-
-  if ($device->is_pseudo) {
-      # always let pseudo devices do contact|location|portname|snapshot
-      # and additionally if there's a snapshot cache, is_discoverable will let
-      # them do all other discover and high prio actions
-      push @badactions, ('discover', grep { $_ !~ m/^(?:contact|location|portname|snapshot)$/ }
-                                          @{ setting('job_prio')->{high} })
-        if not is_discoverable($device);
-  }
-  else {
-      push @badactions, ('discover', @{ setting('job_prio')->{high} })
-        if not is_discoverable($device);
-  }
-
-  push @badactions, (qw/macsuck nbtstat/)
-    if not is_macsuckable($device);
-
-  push @badactions, 'arpnip'
-    if not is_arpnipable($device);
-
-  return @badactions;
-}
-
 sub jq_warm_thrusters {
-  my @devices = schema(vars->{'tenant'})->resultset('Device')->all;
   my $rs = schema(vars->{'tenant'})->resultset('DeviceSkip');
-  my %actionset = ();
-
-  foreach my $d (@devices) {
-    my @badactions = _get_denied_actions($d);
-    $actionset{$d->ip} = \@badactions if scalar @badactions;
-  }
 
   schema(vars->{'tenant'})->txn_do(sub {
     $rs->search({
@@ -86,19 +47,6 @@ sub jq_warm_thrusters {
       actionset => { -value => [] }, # special syntax for matching empty ARRAY
       deferrals => 0,
     })->delete;
-
-    $rs->update_or_create({
-      backend => setting('workers')->{'BACKEND'},
-      device  => $_,
-      actionset => $actionset{$_},
-    }, { key => 'primary' }) for keys %actionset;
-
-    # add one faux record to allow *walk actions to see there is a backend running
-    $rs->update_or_create({
-      backend => setting('workers')->{'BACKEND'},
-      device  => '255.255.255.255',
-      last_defer => \'LOCALTIMESTAMP',
-    }, { key => 'primary' });
   });
 }
 
@@ -122,7 +70,7 @@ sub jq_getsome {
       # and the skiplist was primed. these should be checked against
       # the various acls and have device_skip entry added if needed,
       # and return false if it should have been skipped.
-      my @badactions = _get_denied_actions($job->device);
+      my @badactions = get_denied_actions($job->device);
       if (scalar @badactions) {
         schema(vars->{'tenant'})->resultset('DeviceSkip')->find_or_create({
           backend => setting('workers')->{'BACKEND'}, device => $job->device,
@@ -205,6 +153,7 @@ sub jq_queued {
 
 sub jq_lock {
   my $job = shift;
+  return true unless $job->id;
   my $happy = false;
 
   # lock db row and update to show job has been picked
@@ -251,6 +200,8 @@ sub jq_defer {
         },{ key => 'device_skip_pkey' })->increment_deferrals;
       }
 
+      debug sprintf 'defer: job %s', ($job->id || 'unknown');
+
       # lock db row and update to show job is available
       schema(vars->{'tenant'})->resultset('Admin')
         ->search({ job => $job->id }, { for => 'update' })

lib/App/Netdisco/Util/Device.pm

@@ -17,6 +17,7 @@ our @EXPORT_OK = qw/
   is_discoverable is_discoverable_now
   is_arpnipable   is_arpnipable_now
   is_macsuckable  is_macsuckable_now
+  get_denied_actions
 /;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
@@ -329,4 +330,39 @@ sub is_macsuckable_now {
   return is_macsuckable(@_);
 }
 
+=head2 get_denied_actions( $device )
+
+Checks configured ACLs for the device on this backend and returns list
+of actions which are denied.
+
+=cut
+
+sub get_denied_actions {
+  my $device = shift;
+  my @badactions = ();
+  return @badactions unless $device;
+  $device = get_device($device); # might be no-op but is done in is_* anyway
+
+  if ($device->is_pseudo) {
+      # always let pseudo devices do contact|location|portname|snapshot
+      # and additionally if there's a snapshot cache, is_discoverable will let
+      # them do all other discover and high prio actions
+      push @badactions, ('discover', grep { $_ !~ m/^(?:contact|location|portname|snapshot)$/ }
+                                          @{ setting('job_prio')->{high} })
+        if not is_discoverable($device);
+  }
+  else {
+      push @badactions, ('discover', @{ setting('job_prio')->{high} })
+        if not is_discoverable($device);
+  }
+
+  push @badactions, (qw/macsuck nbtstat/)
+    if not is_macsuckable($device);
+
+  push @badactions, 'arpnip'
+    if not is_arpnipable($device);
+
+  return @badactions;
+}
+
 1;

lib/App/Netdisco/Worker/Plugin/Arpwalk.pm

@@ -7,6 +7,17 @@ use aliased 'App::Netdisco::Worker::Status';
 use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
+register_worker({ phase => 'check' }, sub {
+  return Status->defer("arpwalk skipped: have not yet primed skiplist")
+    unless schema(vars->{'tenant'})->resultset('DeviceSkip')
+      ->search({
+        backend => setting('workers')->{'BACKEND'},
+        device  => '255.255.255.255',
+      })->count();
+
+  return Status->done('Arpwalk is able to run');
+});
+
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 

lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm

@@ -7,6 +7,17 @@ use aliased 'App::Netdisco::Worker::Status';
 use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
+register_worker({ phase => 'check' }, sub {
+  return Status->defer("discoverall skipped: have not yet primed skiplist")
+    unless schema(vars->{'tenant'})->resultset('DeviceSkip')
+      ->search({
+        backend => setting('workers')->{'BACKEND'},
+        device  => '255.255.255.255',
+      })->count();
+
+  return Status->done('Discoverall is able to run');
+});
+
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 

lib/App/Netdisco/Worker/Plugin/Macwalk.pm

@@ -7,6 +7,17 @@ use aliased 'App::Netdisco::Worker::Status';
 use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
+register_worker({ phase => 'check' }, sub {
+  return Status->defer("macwalk skipped: have not yet primed skiplist")
+    unless schema(vars->{'tenant'})->resultset('DeviceSkip')
+      ->search({
+        backend => setting('workers')->{'BACKEND'},
+        device  => '255.255.255.255',
+      })->count();
+
+  return Status->done('Macwalk is able to run');
+});
+
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 

lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm

@@ -7,6 +7,17 @@ use aliased 'App::Netdisco::Worker::Status';
 use App::Netdisco::JobQueue 'jq_insert';
 use Dancer::Plugin::DBIC 'schema';
 
+register_worker({ phase => 'check' }, sub {
+  return Status->defer("nbtwalk skipped: have not yet primed skiplist")
+    unless schema(vars->{'tenant'})->resultset('DeviceSkip')
+      ->search({
+        backend => setting('workers')->{'BACKEND'},
+        device  => '255.255.255.255',
+      })->count();
+
+  return Status->done('Nbtwalk is able to run');
+});
+
 register_worker({ phase => 'main' }, sub {
   my ($job, $workerconf) = @_;
 

lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm

@@ -0,0 +1,60 @@
+package App::Netdisco::Worker::Plugin::PrimeSkiplist;
+
+use Dancer ':syntax';
+use Dancer::Plugin::DBIC 'schema';
+
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::Util::Device 'get_denied_actions';
+use App::Netdisco::Backend::Job;
+
+use Try::Tiny;
+
+register_worker({ phase => 'main' }, sub {
+  my ($job, $workerconf) = @_;
+  my $happy = false;
+
+  my $devices = schema(vars->{'tenant'})->resultset('Device');
+  my $rs = schema(vars->{'tenant'})->resultset('DeviceSkip');
+  my %actionset = ();
+
+  while (my $d = $devices->next) {
+    my @badactions = get_denied_actions($d);
+    $actionset{$d->ip} = \@badactions if scalar @badactions;
+  }
+
+  debug sprintf 'priming device action skip list for %d devices',
+    scalar keys %actionset;
+
+  try {
+    schema(vars->{'tenant'})->txn_do(sub {
+      $rs->update_or_create({
+        backend => setting('workers')->{'BACKEND'},
+        device  => $_,
+        actionset => $actionset{$_},
+      }, { key => 'primary' }) for keys %actionset;
+    });
+
+    # add one faux record to allow *walk actions to see there is a backend running
+    $rs->update_or_create({
+      backend => setting('workers')->{'BACKEND'},
+      device  => '255.255.255.255',
+      last_defer => \'LOCALTIMESTAMP',
+    }, { key => 'primary' });
+
+    $happy = true;
+  }
+  catch {
+    error $_;
+  };
+
+  if ($happy) {
+    return Status->done("Primed device action skip list");
+  }
+  else {
+    return Status->error("Failed to prime device action skip list");
+  }
+});
+
+true;

share/config.yml

@@ -456,6 +456,7 @@ workers:
 
 # this one takes ages
 snapshot_timeout: 1200
+primeskiplist_timeout: 1200
 
 # 50 minutes
 jobs_stale_after: 3000
@@ -485,27 +486,27 @@ schedule:
 
 job_prio:
   high:
-    - contact
+    - 'contact'
-    - hook::exec
+    - 'delete'
-    - hook::http
+    - 'hook::exec'
-    - location
+    - 'hook::http'
-    - portcontrol
+    - 'location'
-    - portname
+    - 'portcontrol'
-    - power
+    - 'portname'
-    - snapshot
+    - 'power'
-    - vlan
+    - 'snapshot'
-    - delete
+    - 'vlan'
   normal:
-    - arpnip
+    - 'arpnip'
-    - arpwalk
+    - 'arpwalk'
-    - discover
+    - 'discover'
-    - discoverall
+    - 'discoverall'
-    - expire
+    - 'expire'
-    - macsuck
+    - 'macsuck'
-    - macwalk
+    - 'macwalk'
-    - nbtstat
+    - 'nbtstat'
-    - nbtwalk
+    - 'nbtwalk'
-    - stats
+    - 'stats'
 
 worker_plugins:
   - 'Internal::SNMPFastDiscover'
@@ -559,6 +560,7 @@ worker_plugins:
   - 'PortControl'
   - 'PortName'
   - 'Power'
+  - 'PrimeSkiplist'
   - 'Psql'
   - 'Renumber'
   - 'Show'
@@ -579,6 +581,10 @@ driver_priority:
 
 deferrable_actions:
   - 'snapshot'
+  - 'nbtwalk'
+  - 'macwalk'
+  - 'arpwalk'
+  - 'discoverall'
 
 # ---------------
 # GraphViz Export