diff --git a/Netdisco/Changes b/Netdisco/Changes index 6f4686a5..0d63f03d 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -1,8 +1,20 @@ +2.008002 - 2013-06-11 + + [ENHANCEMENTS] + + * Initial (bootstrap) discover now queues an arpwalk and macwalk as well + + [BUG FIXES] + + * Fix logging when daemonized + * Remove bad constraint on device_ip table + * Limit queueing of discovery to things which can be discovered + 2.008001 - 2013-06-11 [BUG FIXES] - * more fixes for localenv discovery in scripts + * More fixes for localenv discovery in scripts 2.008000 - 2013-06-09 diff --git a/Netdisco/MANIFEST b/Netdisco/MANIFEST index cc9f2479..3c139589 100644 --- a/Netdisco/MANIFEST +++ b/Netdisco/MANIFEST @@ -26,6 +26,7 @@ lib/App/Netdisco/Daemon/DB.pm lib/App/Netdisco/Daemon/DB/Result/Admin.pm lib/App/Netdisco/Daemon/Queue.pm lib/App/Netdisco/Daemon/Util.pm +lib/App/Netdisco/Daemon/Worker/Common.pm lib/App/Netdisco/Daemon/Worker/Interactive.pm lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm @@ -93,6 +94,7 @@ lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-18-19-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-19-20-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-3-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-PostgreSQL.sql +lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-3-4-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-4-5-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-5-6-PostgreSQL.sql diff --git a/Netdisco/META.yml b/Netdisco/META.yml index 0656075b..e566952a 100644 --- a/Netdisco/META.yml +++ b/Netdisco/META.yml @@ -60,4 +60,4 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: git://git.code.sf.net/p/netdisco/netdisco-ng -version: 2.008001 +version: 2.008002 diff --git a/Netdisco/bin/netdisco-daemon-fg b/Netdisco/bin/netdisco-daemon-fg index e8046468..7773b26d 100755 --- a/Netdisco/bin/netdisco-daemon-fg +++ b/Netdisco/bin/netdisco-daemon-fg @@ -69,8 +69,7 @@ sub build_tasks_list { user_begin => worker_factory('Interactive'), } if setting('workers')->{interactives}; - info sprintf "MCE will load %s tasks: %s Manager, %s Scheduler, %s Poller, %s Interactive", - (scalar @$tasks), + info sprintf "MCE will load: %s Manager, %s Scheduler, %s Poller, %s Interactive", ((setting('workers')->{pollers} or setting('workers')->{interactives}) ? 1 : 0), (setting('housekeeping') ? 1 : 0), (setting('workers')->{pollers} || 0), diff --git a/Netdisco/lib/App/Netdisco.pm b/Netdisco/lib/App/Netdisco.pm index 79fce419..be0c8a48 100644 --- a/Netdisco/lib/App/Netdisco.pm +++ b/Netdisco/lib/App/Netdisco.pm @@ -7,7 +7,7 @@ use 5.010_000; use File::ShareDir 'dist_dir'; use Path::Class; -our $VERSION = '2.008001'; +our $VERSION = '2.008002'; BEGIN { if (not length ($ENV{DANCER_APPDIR} || '') diff --git a/Netdisco/lib/App/Netdisco/Core/Discover.pm b/Netdisco/lib/App/Netdisco/Core/Discover.pm index b3448a97..d0a1213d 100644 --- a/Netdisco/lib/App/Netdisco/Core/Discover.pm +++ b/Netdisco/lib/App/Netdisco/Core/Discover.pm @@ -3,7 +3,7 @@ package App::Netdisco::Core::Discover; use Dancer qw/:syntax :script/; use Dancer::Plugin::DBIC 'schema'; -use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::Device qw/get_device is_discoverable/; use App::Netdisco::Util::DNS ':all'; use NetAddr::IP::Lite ':lower'; use Try::Tiny; @@ -51,7 +51,7 @@ sub store_device { _set_canonical_ip($device, $snmp); my $hostname = hostname_from_ip($device->ip); - $device->dns($hostname) if length $hostname; + $device->dns($hostname) if $hostname; my $localnet = NetAddr::IP::Lite->new('127.0.0.0/8'); # build device aliases suitable for DBIC @@ -791,30 +791,26 @@ The Device database object can be a fresh L object which is not yet stored to the database. Any discovered neighbor unknown to Netdisco will have a C job -immediately queued (subject to the filtering by the C -setting). +immediately queued (subject to the filtering by the C settings). =cut sub discover_new_neighbors { my @to_discover = store_neighbors(@_); - # only enqueue if device is not already discovered, and - # discover_no_type config permits the discovery + # only enqueue if device is not already discovered, + # discover_* config permits the discovery foreach my $neighbor (@to_discover) { my ($ip, $remote_type) = @$neighbor; my $device = get_device($ip); next if $device->in_storage; - if ($remote_type) { - if (scalar grep {$remote_type =~ m/$_/} - @{setting('discover_no_type') || []}) { - debug sprintf - ' queue - %s, type [%s] excluded by discover_no_type', - $ip, $remote_type; - next; - } + if (not is_discoverable($device, $remote_type)) { + debug sprintf + ' queue - %s, type [%s] excluded by discover_* config', + $ip, $remote_type; + next; } # could fail if queued job already exists diff --git a/Netdisco/lib/App/Netdisco/DB.pm b/Netdisco/lib/App/Netdisco/DB.pm index edb43bfc..ee928630 100644 --- a/Netdisco/lib/App/Netdisco/DB.pm +++ b/Netdisco/lib/App/Netdisco/DB.pm @@ -8,7 +8,7 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_namespaces; -our $VERSION = 20; # schema version used for upgrades, keep as integer +our $VERSION = 21; # schema version used for upgrades, keep as integer use Path::Class; use File::Basename; diff --git a/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql new file mode 100644 index 00000000..990d8660 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql @@ -0,0 +1,5 @@ +BEGIN; + +ALTER TABLE device_ip DROP CONSTRAINT "device_ip_alias"; + +COMMIT; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Util.pm b/Netdisco/lib/App/Netdisco/Daemon/Util.pm index a807e181..ddf4ff3b 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Util.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Util.pm @@ -4,10 +4,11 @@ package App::Netdisco::Daemon::Util; use base 'Exporter'; our @EXPORT = (); -our @EXPORT_OK = qw/ job_done job_error /; +our @EXPORT_OK = qw/ job_done job_error job_defer /; our %EXPORT_TAGS = (all => \@EXPORT_OK); sub job_done { return ('done', shift) } sub job_error { return ('error', shift) } +sub job_defer { return ('defer', shift) } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm new file mode 100644 index 00000000..6c4583c6 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm @@ -0,0 +1,86 @@ +package App::Netdisco::Daemon::Worker::Common; + +use Dancer qw/:moose :syntax :script/; +use Dancer::Plugin::DBIC 'schema'; +use Try::Tiny; + +use Role::Tiny; +use namespace::clean; + +requires qw/worker_type worker_name munge_action/; + +sub worker_body { + my $self = shift; + my $wid = $self->wid; + + my $type = $self->worker_type; + my $name = $self->worker_name; + + while (1) { + debug "$type ($wid): asking for a job"; + my $jobs = $self->do('take_jobs', $self->wid, $name); + + foreach my $candidate (@$jobs) { + # create a row object so we can use column accessors + # use the local db schema in case it is accidentally 'stored' + # (will throw an exception) + my $job = schema('daemon')->resultset('Admin') + ->new_result($candidate); + my $jid = $job->job; + + my $target = $self->munge_action($job->action); + next unless $self->can($target); + debug "$type ($wid): can ${target}() for job $jid"; + + # do job + my ($status, $log); + try { + $job->started(scalar localtime); + info sprintf "$type (%s): starting %s job(%s) at %s", + $wid, $target, $jid, $job->started; + ($status, $log) = $self->$target($job); + } + catch { + $status = 'error'; + $log = "error running job: $_"; + $self->sendto('stderr', $log ."\n"); + }; + + $self->close_job($job, $status, $log); + } + + debug "$type ($wid): sleeping now..."; + sleep( setting('workers')->{sleep_time} || 5 ); + } +} + +sub close_job { + my ($self, $job, $status, $log) = @_; + my $type = $self->worker_type; + my $now = scalar localtime; + + info sprintf "$type (%s): wrapping up %s job(%s) - status %s at %s", + $self->wid, $job->action, $job->job, $status, $now; + + # lock db row and either defer or complete the job + try { + if ($status eq 'defer') { + schema('netdisco')->resultset('Admin') + ->find($job->job, {for => 'update'}) + ->update({ status => 'queued' }); + } + else { + schema('netdisco')->resultset('Admin') + ->find($job->job, {for => 'update'}) + ->update({ + status => $status, + log => $log, + started => $job->started, + finished => $now, + }); + } + } + catch { $self->sendto('stderr', "error closing job: $_\n") }; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm index 9ee8c485..f0b13ec4 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm @@ -1,75 +1,17 @@ package App::Netdisco::Daemon::Worker::Interactive; -use Dancer qw/:moose :syntax :script/; -use Dancer::Plugin::DBIC 'schema'; -use Try::Tiny; - use Role::Tiny; use namespace::clean; +# main worker body +with 'App::Netdisco::Daemon::Worker::Common'; + # add dispatch methods for interactive actions with 'App::Netdisco::Daemon::Worker::Interactive::DeviceActions', 'App::Netdisco::Daemon::Worker::Interactive::PortActions'; -sub worker_body { - my $self = shift; - my $wid = $self->wid; - - while (1) { - debug "int ($wid): asking for a job"; - my $jobs = $self->do('take_jobs', $self->wid, 'Interactive'); - - foreach my $candidate (@$jobs) { - # create a row object so we can use column accessors - # use the local db schema in case it is accidentally 'stored' - # (will throw an exception) - my $job = schema('daemon')->resultset('Admin') - ->new_result($candidate); - my $jid = $job->job; - - my $target = 'set_'. $job->action; - next unless $self->can($target); - debug "int ($wid): can ${target}() for job $jid"; - - # do job - my ($status, $log); - try { - $job->started(scalar localtime); - info sprintf "int (%s): starting %s job(%s) at %s", - $wid, $target, $jid, $job->started; - ($status, $log) = $self->$target($job); - } - catch { - $status = 'error'; - $log = "error running job: $_"; - $self->sendto('stderr', $log ."\n"); - }; - - $self->close_job($job, $status, $log); - } - - debug "int ($wid): sleeping now..."; - sleep( setting('workers')->{sleep_time} || 5 ); - } -} - -sub close_job { - my ($self, $job, $status, $log) = @_; - my $now = scalar localtime; - info sprintf "int (%s): wrapping up set_%s job(%s) - status %s at %s", - $self->wid, $job->action, $job->job, $status, $now; - - try { - schema('netdisco')->resultset('Admin') - ->find($job->job, {for => 'update'}) - ->update({ - status => $status, - log => $log, - started => $job->started, - finished => $now, - }); - } - catch { $self->sendto('stderr', "error closing job: $_\n") }; -} +sub worker_type { 'int' } +sub worker_name { 'Interactive' } +sub munge_action { 'set_' . $_[1] } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index b87a2629..dff5a21c 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -95,11 +95,10 @@ sub lock_job { # lock db row and update to show job has been picked try { schema('netdisco')->txn_do(sub { - my $row = schema('netdisco')->resultset('Admin')->find( - {job => $job->job, status => 'queued'}, {for => 'update'} - ); - - $row->update({status => "queued-$fqdn"}); + schema('netdisco')->resultset('Admin')->find( + {job => $job->job, status => 'queued'}, + {for => 'update'} + )->update({ status => "queued-$fqdn" }); }); $happy = 1; }; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm index 22f9d71b..8df43f51 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm @@ -1,77 +1,18 @@ package App::Netdisco::Daemon::Worker::Poller; -use Dancer qw/:moose :syntax :script/; -use Dancer::Plugin::DBIC 'schema'; - -use Try::Tiny; - use Role::Tiny; use namespace::clean; +# main worker body +with 'App::Netdisco::Daemon::Worker::Common'; + # add dispatch methods for poller tasks -with 'App::Netdisco::Daemon::Worker::Poller::Device'; -with 'App::Netdisco::Daemon::Worker::Poller::Arpnip'; -with 'App::Netdisco::Daemon::Worker::Poller::Macsuck'; +with 'App::Netdisco::Daemon::Worker::Poller::Device', + 'App::Netdisco::Daemon::Worker::Poller::Arpnip', + 'App::Netdisco::Daemon::Worker::Poller::Macsuck'; -sub worker_body { - my $self = shift; - my $wid = $self->wid; - - while (1) { - debug "poll ($wid): asking for a job"; - my $jobs = $self->do('take_jobs', $self->wid, 'Poller'); - - foreach my $candidate (@$jobs) { - # create a row object so we can use column accessors - # use the local db schema in case it is accidentally 'stored' - # (will throw an exception) - my $job = schema('daemon')->resultset('Admin') - ->new_result($candidate); - my $jid = $job->job; - my $target = $job->action; - - next unless $self->can($target); - debug "poll ($wid): can ${target}() for job $jid"; - - # do job - my ($status, $log); - try { - $job->started(scalar localtime); - info sprintf "poll (%s): starting %s job(%s) at %s", - $wid, $target, $jid, $job->started; - ($status, $log) = $self->$target($job); - } - catch { - $status = 'error'; - $log = "error running job: $_"; - $self->sendto('stderr', $log ."\n"); - }; - - $self->close_job($job, $status, $log); - } - - debug "poll ($wid): sleeping now..."; - sleep( setting('workers')->{sleep_time} || 5 ); - } -} - -sub close_job { - my ($self, $job, $status, $log) = @_; - my $now = scalar localtime; - info sprintf "poll (%s): wrapping up %s job(%s) - status %s at %s", - $self->wid, $job->action, $job->job, $status, $now; - - try { - schema('netdisco')->resultset('Admin') - ->find($job->job, {for => 'update'}) - ->update({ - status => $status, - log => $log, - started => $job->started, - finished => $now, - }); - } - catch { $self->sendto('stderr', "error closing job: $_\n") }; -} +sub worker_type { 'pol' } +sub worker_name { 'Poller' } +sub munge_action { $_[1] } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm index c9f98b7b..29f0b17c 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm @@ -20,6 +20,16 @@ sub arpwalk { my $devices = schema('netdisco')->resultset('Device')->get_column('ip'); my $jobqueue = schema('netdisco')->resultset('Admin'); + if ($job->subaction and $job->subaction eq 'after-discoverall') { + # make sure there are no incomplete discover jobs queued + my $discover = $jobqueue->search( + { action => 'discover', status => { -like => 'queued%' } } + )->count; + + return job_defer("Deferred arpwalk due to pending discover jobs") + if $discover; + } + schema('netdisco')->txn_do(sub { # clean up user submitted jobs older than 1min, # assuming skew between schedulers' clocks is not greater than 1min diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm index 93cf9105..d0e031b2 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm @@ -50,6 +50,10 @@ sub discover { my $host = NetAddr::IP::Lite->new($job->device); my $device = get_device($host->addr); + if ($device->ip eq '0.0.0.0') { + return job_error("discover failed: no device param (need -d ?)"); + } + if ($device->in_storage and $device->vendor and $device->vendor eq 'netdisco') { return job_done("Skipped discover for pseudo-device $host"); diff --git a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod index 2dde3667..f4f01892 100644 --- a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod +++ b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod @@ -65,13 +65,6 @@ Value: C. Default: C. The log level used by Netdisco. It's useful to see warning messages from the backend poller, as this can highlight broken topology. -=head3 C - -Value: C. Default: C. - -Destination for log messages. Console means standard ouput. When set to -C, the default destination is the C<${HOME}/logs> directory. - =head3 C Value: Format String. Default: C<< '[%P] %L @%D> %m' >>. @@ -411,6 +404,15 @@ Value: Boolean. Default: C. Whether to show a stack trace when an error is caught in the web frontend. +=head3 C + +Value: C. Default: C. + +Destination for log messages. Should usually be C, which does the +right thing when running foreground apps, and is also captured to +C<${HOME}/logs> when running daemonized. Only change this if you know what +you're doing. + =head3 C Value: Settings Tree. diff --git a/Netdisco/lib/App/Netdisco/Util/Device.pm b/Netdisco/lib/App/Netdisco/Util/Device.pm index eeba5f2e..061a7b38 100644 --- a/Netdisco/lib/App/Netdisco/Util/Device.pm +++ b/Netdisco/lib/App/Netdisco/Util/Device.pm @@ -67,11 +67,16 @@ Returns false if the host is not permitted to discover the target device. =cut sub is_discoverable { - my $q = shift; + my ($ip, $remote_type) = @_; + my $device = get_device($ip) or return 0; + + if ($remote_type) { + return 0 if + scalar grep {$remote_type =~ m/$_/} + @{setting('discover_no_type') || []}; + } - my $device = get_device($q) or return 0; my $addr = NetAddr::IP::Lite->new($device->ip); - my $discover_no = setting('discover_no') || []; my $discover_only = setting('discover_only') || []; diff --git a/Netdisco/lib/App/Netdisco/Util/SNMP.pm b/Netdisco/lib/App/Netdisco/Util/SNMP.pm index d4afee09..79bd9441 100644 --- a/Netdisco/lib/App/Netdisco/Util/SNMP.pm +++ b/Netdisco/lib/App/Netdisco/Util/SNMP.pm @@ -154,7 +154,7 @@ sub _build_mibdirs { sub _get_mibdirs_content { my $home = shift; - warning 'Netdisco SNMP work will be really slow - loading ALL MIBs. Please set mibdirs.'; + warning 'Netdisco SNMP work will be slow - loading ALL MIBs. Consider setting mibdirs.'; my @list = map {s|$home/||; $_} grep {-d} glob("$home/*"); return \@list; } diff --git a/Netdisco/lib/App/Netdisco/Web/AdminTask.pm b/Netdisco/lib/App/Netdisco/Web/AdminTask.pm index a7e9211e..f13d7629 100644 --- a/Netdisco/lib/App/Netdisco/Web/AdminTask.pm +++ b/Netdisco/lib/App/Netdisco/Web/AdminTask.pm @@ -16,7 +16,7 @@ sub add_job { } try { - # job might already be in the queue, so this could die + # jobs might already be in the queue, so this could die schema('netdisco')->resultset('Admin')->create({ ($device ? (device => $device->addr) : ()), action => $jobtype, @@ -24,6 +24,23 @@ sub add_job { username => session('user'), userip => request->remote_address, }); + + if (param('extra') and param('extra') eq 'with-walk') { + schema('netdisco')->resultset('Admin')->create({ + action => 'macwalk', + subaction => 'after-discoverall', + status => 'queued', + username => session('user'), + userip => request->remote_address, + }); + schema('netdisco')->resultset('Admin')->create({ + action => 'arpwalk', + subaction => 'after-discoverall', + status => 'queued', + username => session('user'), + userip => request->remote_address, + }); + } }; } diff --git a/Netdisco/share/config.yml b/Netdisco/share/config.yml index 57800308..2485460f 100644 --- a/Netdisco/share/config.yml +++ b/Netdisco/share/config.yml @@ -10,7 +10,7 @@ # ---------------- log: 'warning' -logger: 'file' +logger: 'console' logger_format: '[%P] %L @%D> %m' # ------------ diff --git a/Netdisco/share/views/index.tt b/Netdisco/share/views/index.tt index 23b0be31..10bedca5 100644 --- a/Netdisco/share/views/index.tt +++ b/Netdisco/share/views/index.tt @@ -52,6 +52,7 @@
+