initial implementatin of new job queue with skip info per job

lib/App/Netdisco/DB/Result/Admin.pm

@@ -161,4 +161,12 @@ between the date stamp and time stamp. That is:
 
 sub finished_stamp  { return (shift)->get_column('finished_stamp')  }
 
+=head2 duration
+
+Formatted version of the C<duration> field, accurate to the minute.
+
+=cut
+
+sub duration  { return (shift)->get_column('duration')  }
+
 1;

lib/App/Netdisco/DB/Result/Virtual/JobQueue.pm

@@ -0,0 +1,73 @@
+package App::Netdisco::DB::Result::Virtual::JobQueue;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
+
+__PACKAGE__->table('job_queue');
+__PACKAGE__->result_source_instance->is_virtual(1);
+__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
+  WITH limited_jobs AS
+    (SELECT a.job, max(ds.deferrals) AS max_defer, min(ds.last_defer) AS min_defer,
+            CASE WHEN a.status = 'queued' THEN 100
+                 WHEN a.status LIKE 'queued-%' THEN 80
+                 WHEN a.status = 'error' THEN 60
+                 ELSE 40
+             END AS status_priority,
+            CASE WHEN (a.username IS NOT NULL OR
+                       a.action = ANY (string_to_array(btrim(?, '{"}'), '","')))
+                 THEN 100
+                 ELSE 0
+             END AS job_priority
+       FROM admin a LEFT OUTER JOIN device_skip ds USING (device)
+      WHERE (?::text is NULL or a.device <<= ?)
+   GROUP BY a.job
+   ORDER BY status_priority DESC,
+            job_priority DESC,
+            max_defer ASC NULLS FIRST,
+            min_defer ASC NULLS LAST,
+            a.device, a.action)
+
+  SELECT to_char( entered, 'YYYY-MM-DD HH24:MI' ) AS entered_stamp,
+         username, action, device, subaction, port, status, log,
+         replace( age( finished, started ) ::text, 'mon', 'month' ) AS duration,
+         array(SELECT ('backend',backend,'actionset',actionset,'deferrals',deferrals,'last_defer',EXTRACT(EPOCH FROM last_defer))
+                 FROM device_skip ds
+                WHERE ds.device = a.device) AS skips
+
+    FROM admin a INNER JOIN limited_jobs jobs USING (job)
+ENDSQL
+);
+
+__PACKAGE__->add_columns(
+  "device",
+  { data_type => "inet", is_nullable => 0 },
+  "action",
+  { data_type => "text", is_nullable => 1 },
+  "subaction",
+  { data_type => "text", is_nullable => 1 },
+  "port",
+  { data_type => "text", is_nullable => 1 },
+  "status",
+  { data_type => "text", is_nullable => 1 },
+  "username",
+  { data_type => "text", is_nullable => 1 },
+  "log",
+  { data_type => "text", is_nullable => 1 },
+
+  "skips",
+  { data_type => "[text]", is_nullable => 1 },
+
+  "entered_stamp",
+  { data_type => "text", is_nullable => 1 },
+  "duration",
+  { data_type => "text", is_nullable => 1 },
+);
+
+__PACKAGE__->belongs_to('target', 'App::Netdisco::DB::Result::Device',
+  { 'foreign.ip' => 'self.device' }, { join_type => 'LEFT' } );
+
+1;

lib/App/Netdisco/DB/ResultSet/Admin.pm

@@ -46,6 +46,8 @@ will add the following additional synthesized columns to the result set:
 
 =item finished_stamp
 
+=item duration
+
 =back
 
 =cut
@@ -61,6 +63,7 @@ sub with_times {
           entered_stamp => \"to_char(entered, 'YYYY-MM-DD HH24:MI')",
           started_stamp => \"to_char(started, 'YYYY-MM-DD HH24:MI')",
           finished_stamp => \"to_char(finished, 'YYYY-MM-DD HH24:MI')",
+          duration => \"replace(age(finished, started)::text, 'mon', 'month')",
         },
       });
 }

lib/App/Netdisco/JobQueue/PostgreSQL.pm

@@ -270,11 +270,13 @@ sub jq_complete {
 }
 
 sub jq_log {
-  return schema('netdisco')->resultset('Admin')->search({}, {
+  my $filter = shift;
-    prefetch => 'target',
+  return schema('netdisco')->resultset('Virtual::JobQueue')->search({}, {
-    order_by => { -desc => [qw/entered device action/] },
+    join => 'target',
+    '+columns' => ['target.dns','target.ip'],
+    bind => [ setting('job_prio')->{'high'}, $filter, $filter ],
     rows => 50,
-  })->with_times->hri->all;
+  })->hri->all;
 }
 
 sub jq_userlog {

lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm

@@ -8,6 +8,11 @@ use Dancer::Plugin::Auth::Extensible;
 use App::Netdisco::Web::Plugin;
 use App::Netdisco::JobQueue qw/jq_log jq_delete/;
 
+use utf8;
+use Time::Piece;
+use Text::CSV_XS 'csv';
+use NetAddr::IP::Lite ':lower';
+
 register_admin_task({
   tag => 'jobqueue',
   label => 'Job Queue',
@@ -23,9 +28,61 @@ ajax '/ajax/control/admin/jobqueue/delall' => require_role admin => sub {
 };
 
 ajax '/ajax/content/admin/jobqueue' => require_role admin => sub {
+    my $filter = NetAddr::IP::Lite->new(param('q'));
+    my @data = jq_log($filter);
+
+    foreach my $r (@data) {
+      $r->{qstat}->{acl}   = [];
+      $r->{qstat}->{next}  = [];
+      $r->{qstat}->{fails} = [];
+      next unless ($r->{status} eq 'queued');
+
+      foreach my $s (@{$r->{skips}}) {
+        (my $row = $s) =~ s/(^\(|\)$)//g;
+        next unless $row;
+        my %skip = @{ [@{csv(in => \$row)}]->[0] };
+        next unless scalar keys %skip;
+        $skip{actionset} =~ s/(^{|}$)//g;
+        $skip{actionset} = [@{ csv(in => \$skip{actionset}) }]->[0] || [];
+
+        if ($skip{deferrals}) {
+          unshift @{$r->{qstat}->{fails}}, sprintf '%s connection failure%s from %s',
+            $skip{deferrals}, ($skip{deferrals} > 1 ? 's' : ''), $skip{backend};
+        }
+        else {
+          unshift @{$r->{qstat}->{fails}}, sprintf 'No connection failures from %s',
+            $skip{backend};
+        }
+
+        if (scalar @{$skip{actionset}}
+              and scalar grep {$_ eq $r->{action}} @{$skip{actionset}}) {
+          $r->{qstat}->{badacl} = true;
+          unshift @{$r->{qstat}->{acl}}, sprintf 'Blocked by ACL on %s', $skip{backend};
+        }
+        else {
+          push @{$r->{qstat}->{acl}}, sprintf '✔ on %s', $skip{backend};
+        }
+
+        if ($skip{deferrals} >= setting('workers')->{'max_deferrals'}) {
+          $r->{qstat}->{last_defer} = true;
+          my $after = (localtime($skip{last_defer}) + setting('workers')->{'retry_after'});
+          unshift @{$r->{qstat}->{next}}, sprintf 'Will retry after %s on %s',
+            $after->cdate, $skip{backend};
+        }
+        elsif ($skip{deferrals} > 0) {
+          $r->{qstat}->{last_defer} = true;
+          unshift @{$r->{qstat}->{next}}, sprintf 'Connect failed at %s on %s',
+            localtime($skip{last_defer})->cdate, $skip{backend};
+        }
+        else {
+          push @{$r->{qstat}->{next}}, sprintf '✔ on %s', $skip{backend};
+        }
+      }
+    }
+
     content_type('text/html');
     template 'ajax/admintask/jobqueue.tt', {
-      results => [ jq_log ],
+      results => \@data,
     }, { layout => undef };
 };
 

share/public/css/netdisco.css

@@ -454,6 +454,10 @@ td > form.nd_inline-form {
 .tooltip-wrapper .input[disabled] {
   pointer-events: none;
 }
+.tooltip-inner {
+  max-width: none;
+  white-space: nowrap;
+}
 
 /* vlan entry box for netmap */
 #nd_vlan-label {

share/views/admintask.tt

@@ -30,10 +30,10 @@
         href="#[% task.tag %]_pane">[% task.label %]</a></li>
       [% IF task.tag == 'jobqueue' %]
       <span id="nd_device-name">
-        <a class="nd_adminbutton" name="delall" href="#"><i class="icon-trash text-error"></i></a>
+        <a class="nd_adminbutton" name="delall" href="#"><i class="icon-trash icon-large text-error"></i></a>
-        <a id="nd_countdown-refresh" href="#"><i class="text-success icon-refresh"></i></a>
+        <a id="nd_countdown-refresh" href="#"><i class="text-success icon-large icon-refresh"></i></a>
         <a id="nd_countdown-control" href="#">
-          <i id="nd_countdown-control-icon" class="text-success icon-play"></i></a>
+          <i id="nd_countdown-control-icon" class="text-success icon-large icon-play"></i></a>
         <span id="nd_countdown"></span>
       </span>
       [% ELSIF task.tag == 'userlog' %]

share/views/ajax/admintask/jobqueue.tt

@@ -5,14 +5,12 @@
   <thead>
     <tr>
       <th class="nd_center-cell">Entered</th>
-      <th class="nd_center-cell">Action</th>
-      <th class="nd_center-cell">Status</th>
-      <th class="nd_center-cell">Device</th>
-      <th class="nd_center-cell">Port</th>
-      <th class="nd_center-cell">Param</th>
       <th class="nd_center-cell">User</th>
-      <th class="nd_center-cell">Started</th>
+      <th class="nd_center-cell">Action</th>
-      <th class="nd_center-cell">Finished</th>
+      <th class="nd_center-cell">Device</th>
+      <th class="nd_center-cell">Parameters</th>
+      <th class="nd_center-cell">Status</th>
+      <th class="nd_center-cell">Duration</th>
       <th class="nd_center-cell">Action</th>
     </tr>
   </thead>
@@ -25,16 +23,12 @@
       data-content="<pre>[% row.log | html_entity %]</pre>"
     >
       <td class="nd_center-cell">[% row.entered_stamp | html_entity %]</td>
+      <td class="nd_center-cell">[% row.username | html_entity %]</td>
       <td class="nd_center-cell">
         [% FOREACH word IN row.action.split('_') %]
         [% word.ucfirst | html_entity %]&nbsp;
         [% END %]
       </td>
-      [% IF row.status.search('^queued-') %]
-      <td class="nd_center-cell">Running on &quot;[% row.status.remove('^queued-') | html_entity %]&quot;</td>
-      [% ELSE %]
-      <td class="nd_center-cell">[% row.status.ucfirst | html_entity %]</td>
-      [% END %]
       <td class="nd_center-cell">
         [% IF row.action == 'discover' AND row.status == 'error' %]
         <a href="[% uri_for('/') %]?device=[% row.device | uri %]">[% row.device | html_entity %]</a>
@@ -42,11 +36,29 @@
         <a href="[% uri_for('/device') %]?q=[% row.device | uri %]">[% row.target.dns || row.device | html_entity %]</a>
         [% END %]
       </td>
-      <td class="nd_center-cell">[% row.port | html_entity %]</td>
+      <td class="nd_center-cell">[% row.port | html_entity %][% ', ' IF row.port AND row.subaction %][% row.subaction | html_entity %]</td>
-      <td class="nd_center-cell">[% row.subaction | html_entity %]</td>
+
-      <td class="nd_center-cell">[% row.username | html_entity %]</td>
+      [% IF row.status.search('^queued-') %]
-      <td class="nd_center-cell">[% row.started_stamp | html_entity %]</td>
+      <td class="nd_center-cell">Running on &quot;[% row.status.remove('^queued-') | html_entity %]&quot;</td>
-      <td class="nd_center-cell">[% row.finished_stamp | html_entity %]</td>
+      [% ELSIF row.status == 'queued' %]
+      <td class="nd_center-cell">
+        <i class="icon-ban-circle icon-large [% row.qstat.badacl ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.acl.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      &nbsp;
+        <i class="icon-time icon-large [% row.qstat.last_defer ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.next.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      &nbsp;
+        <i class="icon-bolt icon-large [% row.qstat.last_defer ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.fails.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      </td>
+      [% ELSE %]
+      <td class="nd_center-cell">[% row.status.ucfirst | html_entity %]</td>
+      [% END %]
+
+      <td class="nd_center-cell">[% row.duration | html_entity %]</td>
       <td class="nd_center-cell">
         <input data-form="del" name="job" type="hidden" value="[% row.job | html_entity %]">
         <button class="btn nd_adminbutton" name="del" type="submit"><i class="icon-trash text-error"></i></button>

share/views/js/admintask.js

@@ -71,8 +71,10 @@
       ,minLength: 0
     });
 
-    // activate modals
+    // activate modals, tooltips and popovers
     $('.nd_modal').modal({show: false});
+    $("[rel=tooltip]").tooltip({live: true});
+    $("[rel=popover]").popover({live: true});
   }
 
   // on load, establish global delegations for now and future