From 21bc8cf63e9d94d0d5576a716a65d73d37a8f280 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 24 Oct 2023 21:49:37 +0100 Subject: [PATCH] backend field in jobqueue (#1114) implements #969 --- lib/App/Netdisco/DB.pm | 2 +- lib/App/Netdisco/DB/Result/Admin.pm | 2 ++ lib/App/Netdisco/DB/Result/Device.pm | 1 - .../DB/Result/Virtual/PollerPerformance.pm | 2 +- .../Netdisco/DB/Result/Virtual/TastyJobs.pm | 1 + .../Netdisco/DB/Result/Virtual/WalkJobs.pm | 1 + lib/App/Netdisco/DB/ResultSet/Device.pm | 1 - lib/App/Netdisco/JobQueue/PostgreSQL.pm | 31 ++++++++++++------- .../Netdisco/Web/Plugin/AdminTask/JobQueue.pm | 12 ++++--- lib/App/Netdisco/Web/TypeAhead.pm | 5 +-- .../App-Netdisco-DB-81-82-PostgreSQL.sql | 5 +++ .../App-Netdisco-DB-82-83-PostgreSQL.sql | 7 +++++ share/views/ajax/admintask/jobqueue.tt | 10 ++---- 13 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 share/schema_versions/App-Netdisco-DB-81-82-PostgreSQL.sql create mode 100644 share/schema_versions/App-Netdisco-DB-82-83-PostgreSQL.sql diff --git a/lib/App/Netdisco/DB.pm b/lib/App/Netdisco/DB.pm index 67e0099e..27b463d4 100644 --- a/lib/App/Netdisco/DB.pm +++ b/lib/App/Netdisco/DB.pm @@ -11,7 +11,7 @@ __PACKAGE__->load_namespaces( ); our # try to hide from kwalitee - $VERSION = 81; # schema version used for upgrades, keep as integer + $VERSION = 83; # schema version used for upgrades, keep as integer use Path::Class; use File::ShareDir 'dist_dir'; diff --git a/lib/App/Netdisco/DB/Result/Admin.pm b/lib/App/Netdisco/DB/Result/Admin.pm index 71c436f3..bda9d3bd 100644 --- a/lib/App/Netdisco/DB/Result/Admin.pm +++ b/lib/App/Netdisco/DB/Result/Admin.pm @@ -46,6 +46,8 @@ __PACKAGE__->add_columns( { data_type => "boolean", is_nullable => 1 }, "device_key", { data_type => "text", is_nullable => 1 }, + "backend", + { data_type => "text", is_nullable => 1 }, ); diff --git a/lib/App/Netdisco/DB/Result/Device.pm b/lib/App/Netdisco/DB/Result/Device.pm index 4e7d1343..c4c8eae1 100644 --- a/lib/App/Netdisco/DB/Result/Device.pm +++ b/lib/App/Netdisco/DB/Result/Device.pm @@ -371,7 +371,6 @@ sub renumber { $schema->resultset('Admin')->search({ device => $old_ip, - status => { '-not_like' => 'queued-%' }, })->delete; $device->update({ diff --git a/lib/App/Netdisco/DB/Result/Virtual/PollerPerformance.pm b/lib/App/Netdisco/DB/Result/Virtual/PollerPerformance.pm index 600c23f1..5457cec1 100644 --- a/lib/App/Netdisco/DB/Result/Virtual/PollerPerformance.pm +++ b/lib/App/Netdisco/DB/Result/Virtual/PollerPerformance.pm @@ -24,7 +24,7 @@ __PACKAGE__->result_source_instance->view_definition(< 1 - AND SUM( CASE WHEN status LIKE 'queued%' THEN 1 ELSE 0 END ) = 0 + AND SUM( CASE WHEN status = 'queued' THEN 1 ELSE 0 END ) = 0 ORDER BY entered DESC, elapsed DESC LIMIT 30 ENDSQL diff --git a/lib/App/Netdisco/DB/Result/Virtual/TastyJobs.pm b/lib/App/Netdisco/DB/Result/Virtual/TastyJobs.pm index ee3caea0..aa0dbf86 100644 --- a/lib/App/Netdisco/DB/Result/Virtual/TastyJobs.pm +++ b/lib/App/Netdisco/DB/Result/Virtual/TastyJobs.pm @@ -16,6 +16,7 @@ __PACKAGE__->result_source_instance->view_definition(<result_source_instance->view_definition(<resultset('Admin')->search({ device => { '-in' => $devices->as_query }, - status => { '-not_like' => 'queued-%' }, })->delete; $schema->resultset('DeviceSkip')->search( diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index 7f650dd9..6a6c8615 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -112,7 +112,8 @@ sub jq_getsome { job => $job->id, -exists => $jobs->search({ job => { '>' => $job->id }, - status => { -like => 'queued-%' }, + status => 'queued', + backend => { '!=' => undef }, started => \[q/> (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')], %job_properties, })->as_query, @@ -131,7 +132,8 @@ sub jq_getsome { sub jq_locked { my @returned = (); my $rs = schema(vars->{'tenant'})->resultset('Admin')->search({ - status => ('queued-'. setting('workers')->{'BACKEND'}), + status => 'queued', + backend => setting('workers')->{'BACKEND'}, started => \[q/> (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')], }); @@ -147,7 +149,7 @@ sub jq_queued { return schema(vars->{'tenant'})->resultset('Admin')->search({ device => { '!=' => undef}, action => $job_type, - status => { -like => 'queued%' }, + status => 'queued', })->get_column('device')->all; } @@ -161,7 +163,8 @@ sub jq_lock { my $updated = schema(vars->{'tenant'})->resultset('Admin') ->search({ job => $job->id, status => 'queued' }, { for => 'update' }) ->update({ - status => ('queued-'. setting('workers')->{'BACKEND'}), + status => 'queued', + backend => setting('workers')->{'BACKEND'}, started => \"LOCALTIMESTAMP", }); @@ -205,7 +208,7 @@ sub jq_defer { # lock db row and update to show job is available schema(vars->{'tenant'})->resultset('Admin') ->search({ job => $job->id }, { for => 'update' }) - ->update({ status => 'queued', started => undef, log => $job->log }); + ->update({ status => 'queued', backend => undef, started => undef, log => $job->log }); }); $happy = true; } @@ -257,12 +260,7 @@ sub jq_complete { sub jq_log { return schema(vars->{'tenant'})->resultset('Admin')->search({ - (param('backend') ? ( - 'me.status' => { '=' => [ - # FIXME 'done-'. param('backend'), - 'queued-'. param('backend'), - ] }, - ) : ()), + (param('backend') ? ('me.backend' => param('backend')) : ()), (param('action') ? ('me.action' => param('action')) : ()), (param('device') ? ( -or => [ @@ -271,7 +269,16 @@ sub jq_log { ], ) : ()), (param('username') ? ('me.username' => param('username')) : ()), - (param('status') ? ('me.status' => lc(param('status'))) : ()), + (param('status') ? ( + (param('status') eq 'Running') ? ( + -and => [ + { 'me.backend' => { '!=' => undef } }, + { 'me.status' => 'queued' }, + ], + ) : ( + 'me.status' => lc(param('status')) + ) + ) : ()), (param('duration') ? ( -bool => [ -or => [ diff --git a/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm b/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm index ed41d7f6..4dff6b5b 100644 --- a/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm +++ b/lib/App/Netdisco/Web/Plugin/AdminTask/JobQueue.pm @@ -32,10 +32,14 @@ ajax '/ajax/content/admin/jobqueue' => require_role admin => sub { content_type('text/html'); my $jq_total = schema(vars->{'tenant'})->resultset('Admin')->count(); - my $jq_queued = schema(vars->{'tenant'})->resultset('Admin')->search({status => 'queued'})->count(); - my $jq_running = schema(vars->{'tenant'})->resultset('Admin')->search({status => { -like => 'queued-%'}})->count(); - my $jq_done = schema(vars->{'tenant'})->resultset('Admin')->search({status => 'done'})->count(); - my $jq_errored = schema(vars->{'tenant'})->resultset('Admin')->search({status => 'error'})->count(); + my $jq_queued = schema(vars->{'tenant'})->resultset('Admin') + ->search({status => 'queued', backend => undef })->count(); + my $jq_running = schema(vars->{'tenant'})->resultset('Admin') + ->search({status => 'queued', backend => { '!=' => undef }})->count(); + my $jq_done = schema(vars->{'tenant'})->resultset('Admin') + ->search({status => 'done'})->count(); + my $jq_errored = schema(vars->{'tenant'})->resultset('Admin') + ->search({status => 'error'})->count(); template 'ajax/admintask/jobqueue.tt', { jq_total => commify($jq_total || 0), diff --git a/lib/App/Netdisco/Web/TypeAhead.pm b/lib/App/Netdisco/Web/TypeAhead.pm index 694b8351..19a02376 100644 --- a/lib/App/Netdisco/Web/TypeAhead.pm +++ b/lib/App/Netdisco/Web/TypeAhead.pm @@ -87,10 +87,7 @@ ajax '/ajax/data/queue/typeahead/status' => require_role admin => sub { my $q = quotemeta( param('query') || param('term') || param('status') ); my @actions = grep { $q ? m/^$q/ : true } - List::MoreUtils::uniq - sort - grep { defined } - qw(Queued Done Info Deferred Error); + qw(Queued Running Done Info Deferred Error); content_type 'application/json'; to_json \@actions; diff --git a/share/schema_versions/App-Netdisco-DB-81-82-PostgreSQL.sql b/share/schema_versions/App-Netdisco-DB-81-82-PostgreSQL.sql new file mode 100644 index 00000000..4b4cf504 --- /dev/null +++ b/share/schema_versions/App-Netdisco-DB-81-82-PostgreSQL.sql @@ -0,0 +1,5 @@ +BEGIN; + +ALTER TABLE admin ADD COLUMN "backend" text; + +COMMIT; diff --git a/share/schema_versions/App-Netdisco-DB-82-83-PostgreSQL.sql b/share/schema_versions/App-Netdisco-DB-82-83-PostgreSQL.sql new file mode 100644 index 00000000..49591197 --- /dev/null +++ b/share/schema_versions/App-Netdisco-DB-82-83-PostgreSQL.sql @@ -0,0 +1,7 @@ +BEGIN; + +UPDATE admin SET backend = regexp_replace(status, '^queued-', '', '') WHERE status ~ '^queued-'; + +UPDATE admin SET status = 'queued' WHERE status ~ '^queued-'; + +COMMIT; diff --git a/share/views/ajax/admintask/jobqueue.tt b/share/views/ajax/admintask/jobqueue.tt index be4273dc..e780875f 100644 --- a/share/views/ajax/admintask/jobqueue.tt +++ b/share/views/ajax/admintask/jobqueue.tt @@ -26,14 +26,10 @@ - [% IF row.status.search('^queued-') %] - [% row.status.remove('^queued-') | html_entity %] - [% ELSE %] - - [% END %] + [% row.backend | html_entity %] [% FOREACH word IN row.action.split('_') %] @@ -51,7 +47,7 @@ [% row.username | html_entity %] - [% IF row.status.search('^queued-') %] + [% IF (row.status == 'queued' AND row.backend != '') %] Running [% ELSE %] [% row.status.ucfirst | html_entity %]