diff --git a/Netdisco/Changes b/Netdisco/Changes index d1b06489..44cfaa53 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -14,6 +14,7 @@ * Attempt to keep PID and logfile as netdisco user even when running as root * Show netdisco-do docs on options error * Do not leak SNMP community string into debug output (unless SHOW_COMMUNITY=1) + * Process tree updated to show daemon worker status/activity [BUG FIXES] diff --git a/Netdisco/bin/netdisco-daemon b/Netdisco/bin/netdisco-daemon index ee7999f5..6e953c78 100755 --- a/Netdisco/bin/netdisco-daemon +++ b/Netdisco/bin/netdisco-daemon @@ -69,6 +69,7 @@ Daemon::Control->new({ sub restarter { my ($daemon, @program_args) = @_; + $0 = 'netdisco-daemon'; my $child = fork_and_start($daemon, @program_args); exit(1) unless $child; diff --git a/Netdisco/bin/netdisco-daemon-fg b/Netdisco/bin/netdisco-daemon-fg index 0d1073d6..86dea474 100755 --- a/Netdisco/bin/netdisco-daemon-fg +++ b/Netdisco/bin/netdisco-daemon-fg @@ -29,7 +29,6 @@ use NetAddr::IP::Lite ':lower'; use List::Util 'sum'; use Role::Tiny::With; use MCE::Signal '-setpgrp'; -setpgrp(0,0); # only portable variety of setpgrp use MCE; # set temporary MCE files' location in home directory @@ -37,6 +36,9 @@ my $home = ($ENV{NETDISCO_HOME} || $ENV{HOME}); my $tmp_dir = ($ENV{NETDISCO_TEMP} || dir($home, 'tmp')); mkdir $tmp_dir if ! -d $tmp_dir; +setpgrp(0,0); # only portable variety of setpgrp +$0 = 'netdisco-daemon: master'; + my $mce = MCE->new( spawn_delay => 0.15, job_delay => 1.15, @@ -92,6 +94,7 @@ sub worker_factory { return sub { my $self = shift; my $wid = $self->wid; + $0 = sprintf 'netdisco-daemon: worker #%s %s: init', $wid, lc($role); info "applying role $role to worker $wid"; # $self->sendto('stderr', ">>> worker $wid starting with role $role\n"); diff --git a/Netdisco/lib/App/Netdisco/DB/Result/Admin.pm b/Netdisco/lib/App/Netdisco/DB/Result/Admin.pm index 50dd0fa5..0de254ac 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/Admin.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/Admin.pm @@ -56,6 +56,23 @@ __PACKAGE__->set_primary_key("job"); # You can replace this text with custom code or comments, and it will be preserved on regeneration +=head1 METHODS + +=head2 summary + +An attempt to make a meaningful statement about the job. + +=cut + +sub summary { + my $job = shift; + return join ' ', + $job->action, + ($job->device || ''), + ($job->port || ''), + ($job->subaction ? (q{'}. $job->subaction .q{'}) : ''); +} + =head1 ADDITIONAL COLUMNS =head2 entererd_stamp diff --git a/Netdisco/lib/App/Netdisco/Daemon/DB/Result/Admin.pm b/Netdisco/lib/App/Netdisco/Daemon/DB/Result/Admin.pm index 8ad79af8..34cc690c 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/DB/Result/Admin.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/DB/Result/Admin.pm @@ -44,8 +44,44 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("job"); +=head1 METHODS + +=head2 summary + +An attempt to make a meaningful statement about the job. + +=cut + +sub summary { + my $job = shift; + return join ' ', + $job->action, + ($job->device || ''), + ($job->port || ''), + ($job->subaction ? (q{'}. $job->subaction .q{'}) : ''); +} + +=head1 ADDITIONAL COLUMNS + +=head2 extra + +Alias for the C column. + +=cut + sub extra { (shift)->subaction } +=head2 entererd_stamp + +Formatted version of the C field, accurate to the minute. + +The format is somewhat like ISO 8601 or RFC3339 but without the middle C +between the date stamp and time stamp. That is: + + 2012-02-06 12:49 + +=cut + sub entered_stamp { (my $stamp = (shift)->entered) =~ s/\.\d+$//; return $stamp; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm index 8dce49ed..b32901d3 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm @@ -16,6 +16,7 @@ sub worker_body { my $type = $self->worker_type; while (1) { + $0 = sprintf 'netdisco-daemon: worker #%s %s: idle', $wid, lc($type); my $jobs = jq_take($self->wid, $type); foreach my $job (@$jobs) { @@ -23,6 +24,8 @@ sub worker_body { try { $job->started(scalar localtime); + $0 = sprintf 'netdisco-daemon: worker #%s %s: working on #%s: %s', + $wid, lc($type), $job->id, $job->summary; info sprintf "$tag (%s): starting %s job(%s) at %s", $wid, $target, $job->id, $job->started; my ($status, $log) = $self->$target($job); @@ -42,9 +45,12 @@ sub worker_body { sub close_job { my ($self, $job) = @_; - my $tag = $self->worker_tag; - my $now = scalar localtime; + my $tag = $self->worker_tag; + my $type = $self->worker_type; + my $now = scalar localtime; + $0 = sprintf 'netdisco-daemon: worker #%s %s: wrapping up %s #%s: %s', + $self->wid, lc($type), $job->action, $job->id, $job->status; info sprintf "$tag (%s): wrapping up %s job(%s) - status %s at %s", $self->wid, $job->action, $job->id, $job->status, $now; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index 2bf17bad..2e5ad7be 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -11,18 +11,19 @@ use App::Netdisco::JobQueue qw/jq_locked jq_getsome jq_lock/; sub worker_begin { my $self = shift; my $wid = $self->wid; - debug "entering Manager ($wid) worker_begin()"; - if (setting('workers')->{'no_manager'}) { - return debug "mgr ($wid): no need for manager... skip begin"; - } + return debug "mgr ($wid): no need for manager... skip begin" + if setting('workers')->{'no_manager'}; + + debug "entering Manager ($wid) worker_begin()"; # requeue jobs locally debug "mgr ($wid): searching for jobs booked to this processing node"; my @jobs = jq_locked; if (scalar @jobs) { - info sprintf "mgr (%s): found %s jobs booked to this processing node", $wid, scalar @jobs; + info sprintf "mgr (%s): found %s jobs booked to this processing node", + $wid, scalar @jobs; $self->do('add_jobs', @jobs); } } @@ -39,6 +40,7 @@ sub worker_body { while (1) { debug "mgr ($wid): getting potential jobs for $num_slots workers"; + $0 = sprintf 'netdisco-daemon: worker #%s manager: gathering', $wid; # get some pending jobs # TODO also check for stale jobs in Netdisco DB @@ -47,6 +49,7 @@ sub worker_body { # check for available local capacity my $job_type = setting('job_types')->{$job->action}; next unless $job_type and $self->do('capacity_for', $job_type); + debug sprintf "mgr (%s): processing node has capacity for job %s (%s)", $wid, $job->id, $job->action; @@ -60,6 +63,7 @@ sub worker_body { } debug "mgr ($wid): sleeping now..."; + $0 = sprintf 'netdisco-daemon: worker #%s manager: idle', $wid; sleep( setting('workers')->{sleep_time} || 2 ); } } diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm index c6c98671..55f6e34e 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm @@ -35,8 +35,12 @@ sub worker_body { while (1) { # sleep until some point in the next minute my $naptime = 60 - (time % 60) + int(rand(45)); + + $0 = sprintf 'netdisco-daemon: worker #%s scheduler: idle', $wid; debug "sched ($wid): sleeping for $naptime seconds"; + sleep $naptime; + $0 = sprintf 'netdisco-daemon: worker #%s scheduler: queueing', $wid; # NB next_time() returns the next *after* win_start my $win_start = time - (time % 60) - 1;