better job queue stats in web

lib/App/Netdisco/JobQueue/PostgreSQL.pm

@@ -39,6 +39,7 @@ sub jq_warm_thrusters {
     my $deferrals = setting('workers')->{'max_deferrals'} - 1;
     $rs->search({
       backend => setting('workers')->{'BACKEND'},
+      device => { '!=' => '255.255.255.255' },
       deferrals => { '>' => $deferrals },
     }, { for => 'update' }, )->update({ deferrals => $deferrals });
 
@@ -47,6 +48,14 @@ sub jq_warm_thrusters {
       actionset => { -value => [] }, # special syntax for matching empty ARRAY
       deferrals => 0,
     })->delete;
+
+    # also clean out any previous backend hint
+    # primeskiplist action will then run to recreate it
+    $rs->search({
+      backend => setting('workers')->{'BACKEND'},
+      device => '255.255.255.255',
+      actionset => { -value => [] }, # special syntax for matching empty ARRAY
+    })->delete;
   });
 }
 

lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm

@@ -31,22 +31,44 @@ sub commify {
 ajax '/ajax/content/admin/jobqueue' => require_role admin => sub {
     content_type('text/html');
 
-    my $jq_total = schema(vars->{'tenant'})->resultset('Admin')->count();
-    my $jq_queued = schema(vars->{'tenant'})->resultset('Admin')
-      ->search({status => 'queued', backend => undef })->count();
-    my $jq_running = schema(vars->{'tenant'})->resultset('Admin')
+    my @backends = schema(vars->{'tenant'})->resultset('DeviceSkip')
+        ->search({device => '255.255.255.255'})->hri->all;
+
+    my $num_backends = scalar keys @backends;
+    my $tot_workers  = 0;
+    $tot_workers += $_->{deferrals} for @backends;
+
+    my $jq_locked = schema(vars->{'tenant'})->resultset('Admin')
       ->search({status => 'queued', backend => { '!=' => undef }})->count();
+
+    my $jq_backlog = schema(vars->{'tenant'})->resultset('Admin')
+      ->search({status => 'queued', backend => undef })->count();
+
     my $jq_done = schema(vars->{'tenant'})->resultset('Admin')
       ->search({status => 'done'})->count();
+
     my $jq_errored = schema(vars->{'tenant'})->resultset('Admin')
       ->search({status => 'error'})->count();
 
+    my $jq_stale = schema(vars->{'tenant'})->resultset('Admin')->search({
+        status => 'queued',
+        backend => { '!=' => undef },
+        started => \[q/<= (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')],
+    })->count();
+
+    my $jq_total = schema(vars->{'tenant'})->resultset('Admin')->count();
+
     template 'ajax/admintask/jobqueue.tt', {
-      jq_total => commify($jq_total || 0),
-      jq_queued => commify($jq_queued || 0),
-      jq_running => commify($jq_running || 0),
-      jq_done => commify($jq_done || 0),
-      jq_errored => commify($jq_errored || 0),
+      num_backends => commify($num_backends || '?'),
+      tot_workers  => commify($tot_workers || '?'),
+
+      jq_running => commify($jq_locked - $jq_stale),
+      jq_backlog => commify($jq_backlog),
+      jq_done => commify($jq_done),
+      jq_errored => commify($jq_errored),
+      jq_stale => commify($jq_stale),
+      jq_total => commify($jq_total),
+
       results => [ jq_log ],
     }, { layout => undef };
 };

lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm

@@ -7,6 +7,7 @@ use App::Netdisco::Worker::Plugin;
 use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::Util::Device 'get_denied_actions';
+use App::Netdisco::Util::MCE 'parse_max_workers';
 use App::Netdisco::Backend::Job;
 
 use Try::Tiny;
@@ -27,6 +28,8 @@ register_worker({ phase => 'main' }, sub {
   debug sprintf 'priming device action skip list for %d devices',
     scalar keys %actionset;
 
+  my $max_workers = parse_max_workers( setting('workers')->{tasks} ) || 0;
+
   try {
     schema(vars->{'tenant'})->txn_do(sub {
       $rs->update_or_create({
@@ -41,6 +44,7 @@ register_worker({ phase => 'main' }, sub {
       backend => setting('workers')->{'BACKEND'},
       device  => '255.255.255.255',
       last_defer => \'LOCALTIMESTAMP',
+      deferrals => $max_workers,
     }, { key => 'primary' });
 
     $happy = true;

share/views/ajax/admintask/jobqueue.tt

@@ -2,10 +2,13 @@
 <div class="span2 alert alert-info">The job queue is empty.</div>
 [% ELSE %]
 <div class="alert alert-info">
+  Backends: [% num_backends | html_entity %] /
+  Workers: [% tot_workers | html_entity %] /
   Running: [% jq_running | html_entity %] /
-  Queued: [% jq_queued | html_entity %] /
+  Backlog: [% jq_backlog | html_entity %] /
   Done: [% jq_done | html_entity %] /
   Errored: [% jq_errored | html_entity %] /
+  Stale: [% jq_stale | html_entity %] /
   Total: [% jq_total | html_entity %]
 </div>
 <table class="table table-bordered table-condensed table-hover nd_floatinghead">