From 6b80626a296c0de69597c0351f77f4385811f972 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 27 Oct 2023 15:32:06 +0100 Subject: [PATCH] better job queue stats in web --- lib/App/Netdisco/JobQueue/PostgreSQL.pm | 9 +++++ .../Netdisco/Web/Plugin/AdminTask/JobQueue.pm | 40 ++++++++++++++----- .../Netdisco/Worker/Plugin/PrimeSkiplist.pm | 4 ++ share/views/ajax/admintask/jobqueue.tt | 5 ++- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index 6a6c8615..de55a61d 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -39,6 +39,7 @@ sub jq_warm_thrusters { my $deferrals = setting('workers')->{'max_deferrals'} - 1; $rs->search({ backend => setting('workers')->{'BACKEND'}, + device => { '!=' => '255.255.255.255' }, deferrals => { '>' => $deferrals }, }, { for => 'update' }, )->update({ deferrals => $deferrals }); @@ -47,6 +48,14 @@ sub jq_warm_thrusters { actionset => { -value => [] }, # special syntax for matching empty ARRAY deferrals => 0, })->delete; + + # also clean out any previous backend hint + # primeskiplist action will then run to recreate it + $rs->search({ + backend => setting('workers')->{'BACKEND'}, + device => '255.255.255.255', + actionset => { -value => [] }, # special syntax for matching empty ARRAY + })->delete; }); } diff --git a/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm b/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm index 4dff6b5b..014683cc 100644 --- a/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm +++ b/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm @@ -31,22 +31,44 @@ sub commify { ajax '/ajax/content/admin/jobqueue' => require_role admin => sub { content_type('text/html'); - my $jq_total = schema(vars->{'tenant'})->resultset('Admin')->count(); - my $jq_queued = schema(vars->{'tenant'})->resultset('Admin') - ->search({status => 'queued', backend => undef })->count(); - my $jq_running = schema(vars->{'tenant'})->resultset('Admin') + my @backends = schema(vars->{'tenant'})->resultset('DeviceSkip') + ->search({device => '255.255.255.255'})->hri->all; + + my $num_backends = scalar keys @backends; + my $tot_workers = 0; + $tot_workers += $_->{deferrals} for @backends; + + my $jq_locked = schema(vars->{'tenant'})->resultset('Admin') ->search({status => 'queued', backend => { '!=' => undef }})->count(); + + my $jq_backlog = schema(vars->{'tenant'})->resultset('Admin') + ->search({status => 'queued', backend => undef })->count(); + my $jq_done = schema(vars->{'tenant'})->resultset('Admin') ->search({status => 'done'})->count(); + my $jq_errored = schema(vars->{'tenant'})->resultset('Admin') ->search({status => 'error'})->count(); + my $jq_stale = schema(vars->{'tenant'})->resultset('Admin')->search({ + status => 'queued', + backend => { '!=' => undef }, + started => \[q/<= (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')], + })->count(); + + my $jq_total = schema(vars->{'tenant'})->resultset('Admin')->count(); + template 'ajax/admintask/jobqueue.tt', { - jq_total => commify($jq_total || 0), - jq_queued => commify($jq_queued || 0), - jq_running => commify($jq_running || 0), - jq_done => commify($jq_done || 0), - jq_errored => commify($jq_errored || 0), + num_backends => commify($num_backends || '?'), + tot_workers => commify($tot_workers || '?'), + + jq_running => commify($jq_locked - $jq_stale), + jq_backlog => commify($jq_backlog), + jq_done => commify($jq_done), + jq_errored => commify($jq_errored), + jq_stale => commify($jq_stale), + jq_total => commify($jq_total), + results => [ jq_log ], }, { layout => undef }; }; diff --git a/lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm b/lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm index 6137e471..31f0fcee 100644 --- a/lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm +++ b/lib/App/Netdisco/Worker/Plugin/PrimeSkiplist.pm @@ -7,6 +7,7 @@ use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::Util::Device 'get_denied_actions'; +use App::Netdisco::Util::MCE 'parse_max_workers'; use App::Netdisco::Backend::Job; use Try::Tiny; @@ -27,6 +28,8 @@ register_worker({ phase => 'main' }, sub { debug sprintf 'priming device action skip list for %d devices', scalar keys %actionset; + my $max_workers = parse_max_workers( setting('workers')->{tasks} ) || 0; + try { schema(vars->{'tenant'})->txn_do(sub { $rs->update_or_create({ @@ -41,6 +44,7 @@ register_worker({ phase => 'main' }, sub { backend => setting('workers')->{'BACKEND'}, device => '255.255.255.255', last_defer => \'LOCALTIMESTAMP', + deferrals => $max_workers, }, { key => 'primary' }); $happy = true; diff --git a/share/views/ajax/admintask/jobqueue.tt b/share/views/ajax/admintask/jobqueue.tt index e780875f..708c8769 100644 --- a/share/views/ajax/admintask/jobqueue.tt +++ b/share/views/ajax/admintask/jobqueue.tt @@ -2,10 +2,13 @@
The job queue is empty.
[% ELSE %]
+ Backends: [% num_backends | html_entity %] / + Workers: [% tot_workers | html_entity %] / Running: [% jq_running | html_entity %] / - Queued: [% jq_queued | html_entity %] / + Backlog: [% jq_backlog | html_entity %] / Done: [% jq_done | html_entity %] / Errored: [% jq_errored | html_entity %] / + Stale: [% jq_stale | html_entity %] / Total: [% jq_total | html_entity %]