From ea9f140e8f162880b14dd145ab2f10249d5001d8 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 3 Apr 2015 20:14:12 +0100 Subject: [PATCH] [#209] AUTO broken in tasks specification --- Netdisco/Changes | 4 ++++ Netdisco/bin/netdisco-daemon-fg | 6 +++++- .../lib/App/Netdisco/Daemon/Worker/Manager.pm | 10 ++++------ Netdisco/lib/App/Netdisco/Util/Daemon.pm | 19 ++++++++++++++++++- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/Netdisco/Changes b/Netdisco/Changes index 0e5c4026..4b9feee6 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -5,6 +5,10 @@ * Update development doc to mention cpanm installdeps * Update troubleshooting doc to mention four key actions + [BUG FIXES] + + * [#209] AUTO broken in tasks specification + 2.032001 - 2015-03-24 [NEW FEATURES] diff --git a/Netdisco/bin/netdisco-daemon-fg b/Netdisco/bin/netdisco-daemon-fg index f96da845..17e6aebd 100755 --- a/Netdisco/bin/netdisco-daemon-fg +++ b/Netdisco/bin/netdisco-daemon-fg @@ -63,9 +63,13 @@ my $queue = MCE::Queue->new; setting('workers')->{'no_manager'} = 1 if setting('workers')->{tasks} eq '0'; +# MCE::Util has a limit of ncpu if AUTO is used in max_workers, +# so we parse the field ourselves. +my $max_workers = parse_max_workers( setting('workers')->{tasks} ) || 0; + mce_flow { task_name => [qw/ scheduler manager poller /], - max_workers => [ 1, 1, setting('workers')->{tasks} ], + max_workers => [ 1, 1, $max_workers ], tmp_dir => $tmp_dir, on_post_exit => sub { MCE->restart_worker }, }, _mk_wkr('Scheduler'), _mk_wkr('Manager'), _mk_wkr('Poller'); diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index d28f0ff1..604b1b9d 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -44,9 +44,8 @@ sub worker_body { prctl sprintf 'netdisco-daemon: worker #%s manager: gathering', $wid; my $num_slots = 0; - $num_slots = - MCE::Util::_parse_max_workers( setting('workers')->{tasks} ) - - $self->{queue}->pending(); + $num_slots = parse_max_workers( setting('workers')->{tasks} ) + - $self->{queue}->pending(); debug "mgr ($wid): getting potential jobs for $num_slots workers (HP)"; # get some high priority jobs @@ -62,9 +61,8 @@ sub worker_body { $self->{queue}->enqueuep(100, $job); } - $num_slots = - MCE::Util::_parse_max_workers( setting('workers')->{tasks} ) - - $self->{queue}->pending(); + $num_slots = parse_max_workers( setting('workers')->{tasks} ) + - $self->{queue}->pending(); debug "mgr ($wid): getting potential jobs for $num_slots workers (NP)"; # get some normal priority jobs diff --git a/Netdisco/lib/App/Netdisco/Util/Daemon.pm b/Netdisco/lib/App/Netdisco/Util/Daemon.pm index d002a1c8..9234c5b0 100644 --- a/Netdisco/lib/App/Netdisco/Util/Daemon.pm +++ b/Netdisco/lib/App/Netdisco/Util/Daemon.pm @@ -3,11 +3,13 @@ package App::Netdisco::Util::Daemon; use strict; use warnings; +use MCE::Util (); + # make sure this is already done elsewhere use if $^O eq 'linux', 'Sys::Proctitle'; use base 'Exporter'; -our @EXPORT = 'prctl'; +our @EXPORT = qw/prctl parse_max_workers/; sub prctl { if ($^O eq 'linux') { @@ -18,4 +20,19 @@ sub prctl { } } +sub parse_max_workers { + my $max = shift; + return 0 if !defined $max; + + if ($max =~ /^auto(?:$|\s*([\-\+\/\*])\s*(.+)$)/i) { + my $ncpu = MCE::Util::get_ncpu() || 0; + + if ($1 and $2) { + local $@; $max = eval "int($ncpu $1 $2 + 0.5)"; + } + } + + return $max || 0; +} + 1;