initial implementatin of new job queue with skip info per job

2018-04-25 21:43:39 +01:00
parent 5eb443438c
commit 7e1982985c
9 changed files with 187 additions and 26 deletions
--- a/lib/App/Netdisco/DB/Result/Admin.pm
+++ b/lib/App/Netdisco/DB/Result/Admin.pm
@@ -161,4 +161,12 @@ between the date stamp and time stamp. That is:

 sub finished_stamp  { return (shift)->get_column('finished_stamp')  }

+=head2 duration
+
+Formatted version of the C<duration> field, accurate to the minute.
+
+=cut
+
+sub duration  { return (shift)->get_column('duration')  }
+
 1;
--- a/lib/App/Netdisco/DB/Result/Virtual/JobQueue.pm
+++ b/lib/App/Netdisco/DB/Result/Virtual/JobQueue.pm
@@ -0,0 +1,73 @@
+package App::Netdisco::DB::Result::Virtual::JobQueue;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
+
+__PACKAGE__->table('job_queue');
+__PACKAGE__->result_source_instance->is_virtual(1);
+__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
+  WITH limited_jobs AS
+    (SELECT a.job, max(ds.deferrals) AS max_defer, min(ds.last_defer) AS min_defer,
+            CASE WHEN a.status = 'queued' THEN 100
+                 WHEN a.status LIKE 'queued-%' THEN 80
+                 WHEN a.status = 'error' THEN 60
+                 ELSE 40
+             END AS status_priority,
+            CASE WHEN (a.username IS NOT NULL OR
+                       a.action = ANY (string_to_array(btrim(?, '{"}'), '","')))
+                 THEN 100
+                 ELSE 0
+             END AS job_priority
+       FROM admin a LEFT OUTER JOIN device_skip ds USING (device)
+      WHERE (?::text is NULL or a.device <<= ?)
+   GROUP BY a.job
+   ORDER BY status_priority DESC,
+            job_priority DESC,
+            max_defer ASC NULLS FIRST,
+            min_defer ASC NULLS LAST,
+            a.device, a.action)
+
+  SELECT to_char( entered, 'YYYY-MM-DD HH24:MI' ) AS entered_stamp,
+         username, action, device, subaction, port, status, log,
+         replace( age( finished, started ) ::text, 'mon', 'month' ) AS duration,
+         array(SELECT ('backend',backend,'actionset',actionset,'deferrals',deferrals,'last_defer',EXTRACT(EPOCH FROM last_defer))
+                 FROM device_skip ds
+                WHERE ds.device = a.device) AS skips
+
+    FROM admin a INNER JOIN limited_jobs jobs USING (job)
+ENDSQL
+);
+
+__PACKAGE__->add_columns(
+  "device",
+  { data_type => "inet", is_nullable => 0 },
+  "action",
+  { data_type => "text", is_nullable => 1 },
+  "subaction",
+  { data_type => "text", is_nullable => 1 },
+  "port",
+  { data_type => "text", is_nullable => 1 },
+  "status",
+  { data_type => "text", is_nullable => 1 },
+  "username",
+  { data_type => "text", is_nullable => 1 },
+  "log",
+  { data_type => "text", is_nullable => 1 },
+
+  "skips",
+  { data_type => "[text]", is_nullable => 1 },
+
+  "entered_stamp",
+  { data_type => "text", is_nullable => 1 },
+  "duration",
+  { data_type => "text", is_nullable => 1 },
+);
+
+__PACKAGE__->belongs_to('target', 'App::Netdisco::DB::Result::Device',
+  { 'foreign.ip' => 'self.device' }, { join_type => 'LEFT' } );
+
+1;
--- a/lib/App/Netdisco/DB/ResultSet/Admin.pm
+++ b/lib/App/Netdisco/DB/ResultSet/Admin.pm
@@ -46,6 +46,8 @@ will add the following additional synthesized columns to the result set:

 =item finished_stamp

+=item duration
+
 =back

 =cut
@@ -61,6 +63,7 @@ sub with_times {
          entered_stamp => \"to_char(entered, 'YYYY-MM-DD HH24:MI')",
          started_stamp => \"to_char(started, 'YYYY-MM-DD HH24:MI')",
          finished_stamp => \"to_char(finished, 'YYYY-MM-DD HH24:MI')",
+          duration => \"replace(age(finished, started)::text, 'mon', 'month')",
        },
      });
 }
--- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm
+++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm
@@ -270,11 +270,13 @@ sub jq_complete {
 }

 sub jq_log {
-  return schema('netdisco')->resultset('Admin')->search({}, {
-    prefetch => 'target',
-    order_by => { -desc => [qw/entered device action/] },
+  my $filter = shift;
+  return schema('netdisco')->resultset('Virtual::JobQueue')->search({}, {
+    join => 'target',
+    '+columns' => ['target.dns','target.ip'],
+    bind => [ setting('job_prio')->{'high'}, $filter, $filter ],
    rows => 50,
-  })->with_times->hri->all;
+  })->hri->all;
 }

 sub jq_userlog {
--- a/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm
+++ b/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm
@@ -8,6 +8,11 @@ use Dancer::Plugin::Auth::Extensible;
 use App::Netdisco::Web::Plugin;
 use App::Netdisco::JobQueue qw/jq_log jq_delete/;

+use utf8;
+use Time::Piece;
+use Text::CSV_XS 'csv';
+use NetAddr::IP::Lite ':lower';
+
 register_admin_task({
  tag => 'jobqueue',
  label => 'Job Queue',
@@ -23,9 +28,61 @@ ajax '/ajax/control/admin/jobqueue/delall' => require_role admin => sub {
 };

 ajax '/ajax/content/admin/jobqueue' => require_role admin => sub {
+    my $filter = NetAddr::IP::Lite->new(param('q'));
+    my @data = jq_log($filter);
+
+    foreach my $r (@data) {
+      $r->{qstat}->{acl}   = [];
+      $r->{qstat}->{next}  = [];
+      $r->{qstat}->{fails} = [];
+      next unless ($r->{status} eq 'queued');
+
+      foreach my $s (@{$r->{skips}}) {
+        (my $row = $s) =~ s/(^\(|\)$)//g;
+        next unless $row;
+        my %skip = @{ [@{csv(in => \$row)}]->[0] };
+        next unless scalar keys %skip;
+        $skip{actionset} =~ s/(^{|}$)//g;
+        $skip{actionset} = [@{ csv(in => \$skip{actionset}) }]->[0] || [];
+
+        if ($skip{deferrals}) {
+          unshift @{$r->{qstat}->{fails}}, sprintf '%s connection failure%s from %s',
+            $skip{deferrals}, ($skip{deferrals} > 1 ? 's' : ''), $skip{backend};
+        }
+        else {
+          unshift @{$r->{qstat}->{fails}}, sprintf 'No connection failures from %s',
+            $skip{backend};
+        }
+
+        if (scalar @{$skip{actionset}}
+              and scalar grep {$_ eq $r->{action}} @{$skip{actionset}}) {
+          $r->{qstat}->{badacl} = true;
+          unshift @{$r->{qstat}->{acl}}, sprintf 'Blocked by ACL on %s', $skip{backend};
+        }
+        else {
+          push @{$r->{qstat}->{acl}}, sprintf '✔ on %s', $skip{backend};
+        }
+
+        if ($skip{deferrals} >= setting('workers')->{'max_deferrals'}) {
+          $r->{qstat}->{last_defer} = true;
+          my $after = (localtime($skip{last_defer}) + setting('workers')->{'retry_after'});
+          unshift @{$r->{qstat}->{next}}, sprintf 'Will retry after %s on %s',
+            $after->cdate, $skip{backend};
+        }
+        elsif ($skip{deferrals} > 0) {
+          $r->{qstat}->{last_defer} = true;
+          unshift @{$r->{qstat}->{next}}, sprintf 'Connect failed at %s on %s',
+            localtime($skip{last_defer})->cdate, $skip{backend};
+        }
+        else {
+          push @{$r->{qstat}->{next}}, sprintf '✔ on %s', $skip{backend};
+        }
+      }
+    }
+
    content_type('text/html');
    template 'ajax/admintask/jobqueue.tt', {
-      results => [ jq_log ],
+      results => \@data,
    }, { layout => undef };
 };

--- a/share/public/css/netdisco.css
+++ b/share/public/css/netdisco.css
@@ -454,6 +454,10 @@ td > form.nd_inline-form {
 .tooltip-wrapper .input[disabled] {
  pointer-events: none;
 }
+.tooltip-inner {
+  max-width: none;
+  white-space: nowrap;
+}

 /* vlan entry box for netmap */
 #nd_vlan-label {
--- a/share/views/admintask.tt
+++ b/share/views/admintask.tt
@@ -30,10 +30,10 @@
        href="#[% task.tag %]_pane">[% task.label %]</a></li>
      [% IF task.tag == 'jobqueue' %]
      <span id="nd_device-name">
-        <a class="nd_adminbutton" name="delall" href="#"><i class="icon-trash text-error"></i></a>
-        <a id="nd_countdown-refresh" href="#"><i class="text-success icon-refresh"></i></a>
+        <a class="nd_adminbutton" name="delall" href="#"><i class="icon-trash icon-large text-error"></i></a>
+        <a id="nd_countdown-refresh" href="#"><i class="text-success icon-large icon-refresh"></i></a>
        <a id="nd_countdown-control" href="#">
-          <i id="nd_countdown-control-icon" class="text-success icon-play"></i></a>
+          <i id="nd_countdown-control-icon" class="text-success icon-large icon-play"></i></a>
        <span id="nd_countdown"></span>
      </span>
      [% ELSIF task.tag == 'userlog' %]
--- a/share/views/ajax/admintask/jobqueue.tt
+++ b/share/views/ajax/admintask/jobqueue.tt
@@ -5,14 +5,12 @@
  <thead>
    <tr>
      <th class="nd_center-cell">Entered</th>
-      <th class="nd_center-cell">Action</th>
-      <th class="nd_center-cell">Status</th>
-      <th class="nd_center-cell">Device</th>
-      <th class="nd_center-cell">Port</th>
-      <th class="nd_center-cell">Param</th>
      <th class="nd_center-cell">User</th>
-      <th class="nd_center-cell">Started</th>
-      <th class="nd_center-cell">Finished</th>
+      <th class="nd_center-cell">Action</th>
+      <th class="nd_center-cell">Device</th>
+      <th class="nd_center-cell">Parameters</th>
+      <th class="nd_center-cell">Status</th>
+      <th class="nd_center-cell">Duration</th>
      <th class="nd_center-cell">Action</th>
    </tr>
  </thead>
@@ -25,16 +23,12 @@
      data-content="<pre>[% row.log | html_entity %]</pre>"
    >
      <td class="nd_center-cell">[% row.entered_stamp | html_entity %]</td>
+      <td class="nd_center-cell">[% row.username | html_entity %]</td>
      <td class="nd_center-cell">
        [% FOREACH word IN row.action.split('_') %]
        [% word.ucfirst | html_entity %]&nbsp;
        [% END %]
      </td>
-      [% IF row.status.search('^queued-') %]
-      <td class="nd_center-cell">Running on &quot;[% row.status.remove('^queued-') | html_entity %]&quot;</td>
-      [% ELSE %]
-      <td class="nd_center-cell">[% row.status.ucfirst | html_entity %]</td>
-      [% END %]
      <td class="nd_center-cell">
        [% IF row.action == 'discover' AND row.status == 'error' %]
        <a href="[% uri_for('/') %]?device=[% row.device | uri %]">[% row.device | html_entity %]</a>
@@ -42,11 +36,29 @@
        <a href="[% uri_for('/device') %]?q=[% row.device | uri %]">[% row.target.dns || row.device | html_entity %]</a>
        [% END %]
      </td>
-      <td class="nd_center-cell">[% row.port | html_entity %]</td>
-      <td class="nd_center-cell">[% row.subaction | html_entity %]</td>
-      <td class="nd_center-cell">[% row.username | html_entity %]</td>
-      <td class="nd_center-cell">[% row.started_stamp | html_entity %]</td>
-      <td class="nd_center-cell">[% row.finished_stamp | html_entity %]</td>
+      <td class="nd_center-cell">[% row.port | html_entity %][% ', ' IF row.port AND row.subaction %][% row.subaction | html_entity %]</td>
+
+      [% IF row.status.search('^queued-') %]
+      <td class="nd_center-cell">Running on &quot;[% row.status.remove('^queued-') | html_entity %]&quot;</td>
+      [% ELSIF row.status == 'queued' %]
+      <td class="nd_center-cell">
+        <i class="icon-ban-circle icon-large [% row.qstat.badacl ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.acl.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      &nbsp;
+        <i class="icon-time icon-large [% row.qstat.last_defer ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.next.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      &nbsp;
+        <i class="icon-bolt icon-large [% row.qstat.last_defer ? 'text-error' : 'text-success' %]"
+          rel="tooltip" data-placement="bottom" data-offset="5" data-title="[% row.qstat.fails.join('<br>') %]"
+          data-container="body" data-html="true"></i>
+      </td>
+      [% ELSE %]
+      <td class="nd_center-cell">[% row.status.ucfirst | html_entity %]</td>
+      [% END %]
+
+      <td class="nd_center-cell">[% row.duration | html_entity %]</td>
      <td class="nd_center-cell">
        <input data-form="del" name="job" type="hidden" value="[% row.job | html_entity %]">
        <button class="btn nd_adminbutton" name="del" type="submit"><i class="icon-trash text-error"></i></button>
--- a/share/views/js/admintask.js
+++ b/share/views/js/admintask.js
@@ -71,8 +71,10 @@
      ,minLength: 0
    });

-    // activate modals
+    // activate modals, tooltips and popovers
    $('.nd_modal').modal({show: false});
+    $("[rel=tooltip]").tooltip({live: true});
+    $("[rel=popover]").popover({live: true});
  }

  // on load, establish global delegations for now and future