135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit c8f399b2d70248acde6bbabfa58ed4312bf1f8c2
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:53:14 2013 +0100

    version bump

commit 39d16adf38c142feeb1378c217a59783c5a12b44
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:43:37 2013 +0100

    initial discover triggers *walk

commit 29c334db714ac4846d024d815ffe1822889e879c
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:07:03 2013 +0100

    factor out body of poller and interactive worker role

commit db21f88e5a24ad99d808e70213cab1d83cc1ffee
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 20:58:07 2013 +0100

    fix logging when daemonized

commit 1f4a0539de7368273ddc7c24a8ffb0a663817e72
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 17:49:44 2013 +0100

    limit discovery and remove duff constraint on device_ip
This commit is contained in:
Oliver Gorwits
2013-06-11 22:54:16 +01:00
parent 0bff56f729
commit 48b0758840
21 changed files with 193 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Netdisco/Changes
View File

@@ -1,8 +1,20 @@
2.008002 - 2013-06-11
[ENHANCEMENTS]
* Initial (bootstrap) discover now queues an arpwalk and macwalk as well
[BUG FIXES]
* Fix logging when daemonized
* Remove bad constraint on device_ip table
* Limit queueing of discovery to things which can be discovered
2.008001 - 2013-06-11
[BUG FIXES]
* more fixes for localenv discovery in scripts
* More fixes for localenv discovery in scripts
2.008000 - 2013-06-09

2
Netdisco/MANIFEST
View File

@@ -26,6 +26,7 @@ lib/App/Netdisco/Daemon/DB.pm
lib/App/Netdisco/Daemon/DB/Result/Admin.pm
lib/App/Netdisco/Daemon/Queue.pm
lib/App/Netdisco/Daemon/Util.pm
lib/App/Netdisco/Daemon/Worker/Common.pm
lib/App/Netdisco/Daemon/Worker/Interactive.pm
lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm
lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm
@@ -93,6 +94,7 @@ lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-18-19-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-19-20-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-3-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-3-4-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-4-5-PostgreSQL.sql
lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-5-6-PostgreSQL.sql

2
Netdisco/META.yml
View File

@@ -60,4 +60,4 @@ resources:
homepage: http://netdisco.org/
license: http://opensource.org/licenses/bsd-license.php
repository: git://git.code.sf.net/p/netdisco/netdisco-ng
version: 2.008001
version: 2.008002

3
Netdisco/bin/netdisco-daemon-fg
View File

@@ -69,8 +69,7 @@ sub build_tasks_list {
user_begin => worker_factory('Interactive'),
} if setting('workers')->{interactives};
info sprintf "MCE will load %s tasks: %s Manager, %s Scheduler, %s Poller, %s Interactive",
(scalar @$tasks),
info sprintf "MCE will load: %s Manager, %s Scheduler, %s Poller, %s Interactive",
((setting('workers')->{pollers} or setting('workers')->{interactives}) ? 1 : 0),
(setting('housekeeping') ? 1 : 0),
(setting('workers')->{pollers} || 0),

2
Netdisco/lib/App/Netdisco.pm
View File

@@ -7,7 +7,7 @@ use 5.010_000;
use File::ShareDir 'dist_dir';
use Path::Class;
our $VERSION = '2.008001';
our $VERSION = '2.008002';
BEGIN {
if (not length ($ENV{DANCER_APPDIR} || '')

24
Netdisco/lib/App/Netdisco/Core/Discover.pm
View File

@@ -3,7 +3,7 @@ package App::Netdisco::Core::Discover;
use Dancer qw/:syntax :script/;
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Util::Device 'get_device';
use App::Netdisco::Util::Device qw/get_device is_discoverable/;
use App::Netdisco::Util::DNS ':all';
use NetAddr::IP::Lite ':lower';
use Try::Tiny;
@@ -51,7 +51,7 @@ sub store_device {
_set_canonical_ip($device, $snmp);
my $hostname = hostname_from_ip($device->ip);
$device->dns($hostname) if length $hostname;
$device->dns($hostname) if $hostname;
my $localnet = NetAddr::IP::Lite->new('127.0.0.0/8');
# build device aliases suitable for DBIC
@@ -791,30 +791,26 @@ The Device database object can be a fresh L<DBIx::Class::Row> object which is
not yet stored to the database.
Any discovered neighbor unknown to Netdisco will have a C<discover> job
immediately queued (subject to the filtering by the C<discover_no_type>
setting).
immediately queued (subject to the filtering by the C<discover_*> settings).
=cut
sub discover_new_neighbors {
my @to_discover = store_neighbors(@_);
# only enqueue if device is not already discovered, and
# discover_no_type config permits the discovery
# only enqueue if device is not already discovered,
# discover_* config permits the discovery
foreach my $neighbor (@to_discover) {
my ($ip, $remote_type) = @$neighbor;
my $device = get_device($ip);
next if $device->in_storage;
if ($remote_type) {
if (scalar grep {$remote_type =~ m/$_/}
@{setting('discover_no_type') || []}) {
debug sprintf
' queue - %s, type [%s] excluded by discover_no_type',
$ip, $remote_type;
next;
}
if (not is_discoverable($device, $remote_type)) {
debug sprintf
' queue - %s, type [%s] excluded by discover_* config',
$ip, $remote_type;
next;
}
# could fail if queued job already exists

2
Netdisco/lib/App/Netdisco/DB.pm
View File

@@ -8,7 +8,7 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_namespaces;
our $VERSION = 20; # schema version used for upgrades, keep as integer
our $VERSION = 21; # schema version used for upgrades, keep as integer
use Path::Class;
use File::Basename;

5
Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql Normal file
View File

@@ -0,0 +1,5 @@
BEGIN;
ALTER TABLE device_ip DROP CONSTRAINT "device_ip_alias";
COMMIT;

3
Netdisco/lib/App/Netdisco/Daemon/Util.pm
View File

@@ -4,10 +4,11 @@ package App::Netdisco::Daemon::Util;
use base 'Exporter';
our @EXPORT = ();
our @EXPORT_OK = qw/ job_done job_error /;
our @EXPORT_OK = qw/ job_done job_error job_defer /;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
sub job_done { return ('done', shift) }
sub job_error { return ('error', shift) }
sub job_defer { return ('defer', shift) }
1;

86
Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm Normal file
View File

@@ -0,0 +1,86 @@
package App::Netdisco::Daemon::Worker::Common;
use Dancer qw/:moose :syntax :script/;
use Dancer::Plugin::DBIC 'schema';
use Try::Tiny;
use Role::Tiny;
use namespace::clean;
requires qw/worker_type worker_name munge_action/;
sub worker_body {
my $self = shift;
my $wid = $self->wid;
my $type = $self->worker_type;
my $name = $self->worker_name;
while (1) {
debug "$type ($wid): asking for a job";
my $jobs = $self->do('take_jobs', $self->wid, $name);
foreach my $candidate (@$jobs) {
# create a row object so we can use column accessors
# use the local db schema in case it is accidentally 'stored'
# (will throw an exception)
my $job = schema('daemon')->resultset('Admin')
->new_result($candidate);
my $jid = $job->job;
my $target = $self->munge_action($job->action);
next unless $self->can($target);
debug "$type ($wid): can ${target}() for job $jid";
# do job
my ($status, $log);
try {
$job->started(scalar localtime);
info sprintf "$type (%s): starting %s job(%s) at %s",
$wid, $target, $jid, $job->started;
($status, $log) = $self->$target($job);
}
catch {
$status = 'error';
$log = "error running job: $_";
$self->sendto('stderr', $log ."\n");
};
$self->close_job($job, $status, $log);
}
debug "$type ($wid): sleeping now...";
sleep( setting('workers')->{sleep_time} || 5 );
}
}
sub close_job {
my ($self, $job, $status, $log) = @_;
my $type = $self->worker_type;
my $now = scalar localtime;
info sprintf "$type (%s): wrapping up %s job(%s) - status %s at %s",
$self->wid, $job->action, $job->job, $status, $now;
# lock db row and either defer or complete the job
try {
if ($status eq 'defer') {
schema('netdisco')->resultset('Admin')
->find($job->job, {for => 'update'})
->update({ status => 'queued' });
}
else {
schema('netdisco')->resultset('Admin')
->find($job->job, {for => 'update'})
->update({
status => $status,
log => $log,
started => $job->started,
finished => $now,
});
}
}
catch { $self->sendto('stderr', "error closing job: $_\n") };
}
1;

70
Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm
View File

@@ -1,75 +1,17 @@
package App::Netdisco::Daemon::Worker::Interactive;
use Dancer qw/:moose :syntax :script/;
use Dancer::Plugin::DBIC 'schema';
use Try::Tiny;
use Role::Tiny;
use namespace::clean;
# main worker body
with 'App::Netdisco::Daemon::Worker::Common';
# add dispatch methods for interactive actions
with 'App::Netdisco::Daemon::Worker::Interactive::DeviceActions',
'App::Netdisco::Daemon::Worker::Interactive::PortActions';
sub worker_body {
my $self = shift;
my $wid = $self->wid;
while (1) {
debug "int ($wid): asking for a job";
my $jobs = $self->do('take_jobs', $self->wid, 'Interactive');
foreach my $candidate (@$jobs) {
# create a row object so we can use column accessors
# use the local db schema in case it is accidentally 'stored'
# (will throw an exception)
my $job = schema('daemon')->resultset('Admin')
->new_result($candidate);
my $jid = $job->job;
my $target = 'set_'. $job->action;
next unless $self->can($target);
debug "int ($wid): can ${target}() for job $jid";
# do job
my ($status, $log);
try {
$job->started(scalar localtime);
info sprintf "int (%s): starting %s job(%s) at %s",
$wid, $target, $jid, $job->started;
($status, $log) = $self->$target($job);
}
catch {
$status = 'error';
$log = "error running job: $_";
$self->sendto('stderr', $log ."\n");
};
$self->close_job($job, $status, $log);
}
debug "int ($wid): sleeping now...";
sleep( setting('workers')->{sleep_time} || 5 );
}
}
sub close_job {
my ($self, $job, $status, $log) = @_;
my $now = scalar localtime;
info sprintf "int (%s): wrapping up set_%s job(%s) - status %s at %s",
$self->wid, $job->action, $job->job, $status, $now;
try {
schema('netdisco')->resultset('Admin')
->find($job->job, {for => 'update'})
->update({
status => $status,
log => $log,
started => $job->started,
finished => $now,
});
}
catch { $self->sendto('stderr', "error closing job: $_\n") };
}
sub worker_type { 'int' }
sub worker_name { 'Interactive' }
sub munge_action { 'set_' . $_[1] }
1;

9
Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm
View File

@@ -95,11 +95,10 @@ sub lock_job {
# lock db row and update to show job has been picked
try {
schema('netdisco')->txn_do(sub {
my $row = schema('netdisco')->resultset('Admin')->find(
{job => $job->job, status => 'queued'}, {for => 'update'}
);
$row->update({status => "queued-$fqdn"});
schema('netdisco')->resultset('Admin')->find(
{job => $job->job, status => 'queued'},
{for => 'update'}
)->update({ status => "queued-$fqdn" });
});
$happy = 1;
};

77
Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm
View File

@@ -1,77 +1,18 @@
package App::Netdisco::Daemon::Worker::Poller;
use Dancer qw/:moose :syntax :script/;
use Dancer::Plugin::DBIC 'schema';
use Try::Tiny;
use Role::Tiny;
use namespace::clean;
# main worker body
with 'App::Netdisco::Daemon::Worker::Common';
# add dispatch methods for poller tasks
with 'App::Netdisco::Daemon::Worker::Poller::Device';
with 'App::Netdisco::Daemon::Worker::Poller::Arpnip';
with 'App::Netdisco::Daemon::Worker::Poller::Macsuck';
with 'App::Netdisco::Daemon::Worker::Poller::Device',
'App::Netdisco::Daemon::Worker::Poller::Arpnip',
'App::Netdisco::Daemon::Worker::Poller::Macsuck';
sub worker_body {
my $self = shift;
my $wid = $self->wid;
while (1) {
debug "poll ($wid): asking for a job";
my $jobs = $self->do('take_jobs', $self->wid, 'Poller');
foreach my $candidate (@$jobs) {
# create a row object so we can use column accessors
# use the local db schema in case it is accidentally 'stored'
# (will throw an exception)
my $job = schema('daemon')->resultset('Admin')
->new_result($candidate);
my $jid = $job->job;
my $target = $job->action;
next unless $self->can($target);
debug "poll ($wid): can ${target}() for job $jid";
# do job
my ($status, $log);
try {
$job->started(scalar localtime);
info sprintf "poll (%s): starting %s job(%s) at %s",
$wid, $target, $jid, $job->started;
($status, $log) = $self->$target($job);
}
catch {
$status = 'error';
$log = "error running job: $_";
$self->sendto('stderr', $log ."\n");
};
$self->close_job($job, $status, $log);
}
debug "poll ($wid): sleeping now...";
sleep( setting('workers')->{sleep_time} || 5 );
}
}
sub close_job {
my ($self, $job, $status, $log) = @_;
my $now = scalar localtime;
info sprintf "poll (%s): wrapping up %s job(%s) - status %s at %s",
$self->wid, $job->action, $job->job, $status, $now;
try {
schema('netdisco')->resultset('Admin')
->find($job->job, {for => 'update'})
->update({
status => $status,
log => $log,
started => $job->started,
finished => $now,
});
}
catch { $self->sendto('stderr', "error closing job: $_\n") };
}
sub worker_type { 'pol' }
sub worker_name { 'Poller' }
sub munge_action { $_[1] }
1;

10
Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm
View File

@@ -20,6 +20,16 @@ sub arpwalk {
my $devices = schema('netdisco')->resultset('Device')->get_column('ip');
my $jobqueue = schema('netdisco')->resultset('Admin');
if ($job->subaction and $job->subaction eq 'after-discoverall') {
# make sure there are no incomplete discover jobs queued
my $discover = $jobqueue->search(
{ action => 'discover', status => { -like => 'queued%' } }
)->count;
return job_defer("Deferred arpwalk due to pending discover jobs")
if $discover;
}
schema('netdisco')->txn_do(sub {
# clean up user submitted jobs older than 1min,
# assuming skew between schedulers' clocks is not greater than 1min

4
Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm
View File

@@ -50,6 +50,10 @@ sub discover {
my $host = NetAddr::IP::Lite->new($job->device);
my $device = get_device($host->addr);
if ($device->ip eq '0.0.0.0') {
return job_error("discover failed: no device param (need -d ?)");
}
if ($device->in_storage
and $device->vendor and $device->vendor eq 'netdisco') {
return job_done("Skipped discover for pseudo-device $host");

16
Netdisco/lib/App/Netdisco/Manual/Configuration.pod
View File

@@ -65,13 +65,6 @@ Value: C<debug|warning|error>. Default: C<warning>.
The log level used by Netdisco. It's useful to see warning messages from the
backend poller, as this can highlight broken topology.
=head3 C<logger>
Value: C<console|file>. Default: C<file>.
Destination for log messages. Console means standard ouput. When set to
C<file>, the default destination is the C<${HOME}/logs> directory.
=head3 C<logger_format>
Value: Format String. Default: C<< '[%P] %L @%D> %m' >>.
@@ -411,6 +404,15 @@ Value: Boolean. Default: C<false>.
Whether to show a stack trace when an error is caught in the web frontend.
=head3 C<logger>
Value: C<console|file>. Default: C<console>.
Destination for log messages. Should usually be C<console>, which does the
right thing when running foreground apps, and is also captured to
C<${HOME}/logs> when running daemonized. Only change this if you know what
you're doing.
=head3 C<engines>
Value: Settings Tree.

11
Netdisco/lib/App/Netdisco/Util/Device.pm
View File

@@ -67,11 +67,16 @@ Returns false if the host is not permitted to discover the target device.
=cut
sub is_discoverable {
my $q = shift;
my ($ip, $remote_type) = @_;
my $device = get_device($ip) or return 0;
if ($remote_type) {
return 0 if
scalar grep {$remote_type =~ m/$_/}
@{setting('discover_no_type') || []};
}
my $device = get_device($q) or return 0;
my $addr = NetAddr::IP::Lite->new($device->ip);
my $discover_no = setting('discover_no') || [];
my $discover_only = setting('discover_only') || [];

2
Netdisco/lib/App/Netdisco/Util/SNMP.pm
View File

@@ -154,7 +154,7 @@ sub _build_mibdirs {
sub _get_mibdirs_content {
my $home = shift;
warning 'Netdisco SNMP work will be really slow - loading ALL MIBs. Please set mibdirs.';
warning 'Netdisco SNMP work will be slow - loading ALL MIBs. Consider setting mibdirs.';
my @list = map {s|$home/||; $_} grep {-d} glob("$home/*");
return \@list;
}

19
Netdisco/lib/App/Netdisco/Web/AdminTask.pm
View File

@@ -16,7 +16,7 @@ sub add_job {
}
try {
# job might already be in the queue, so this could die
# jobs might already be in the queue, so this could die
schema('netdisco')->resultset('Admin')->create({
($device ? (device => $device->addr) : ()),
action => $jobtype,
@@ -24,6 +24,23 @@ sub add_job {
username => session('user'),
userip => request->remote_address,
});
if (param('extra') and param('extra') eq 'with-walk') {
schema('netdisco')->resultset('Admin')->create({
action => 'macwalk',
subaction => 'after-discoverall',
status => 'queued',
username => session('user'),
userip => request->remote_address,
});
schema('netdisco')->resultset('Admin')->create({
action => 'arpwalk',
subaction => 'after-discoverall',
status => 'queued',
username => session('user'),
userip => request->remote_address,
});
}
};
}

2
Netdisco/share/config.yml
View File

@@ -10,7 +10,7 @@
# ----------------
log: 'warning'
logger: 'file'
logger: 'console'
logger_format: '[%P] %L @%D> %m'
# ------------

1
Netdisco/share/views/index.tt
View File

@@ -52,6 +52,7 @@
<form method="post" action="[% uri_for('/admin/discover') %]">
<div class="form-horizontal">
<input placeholder="Device hostname or IP" class="span4" name="device" type="text"/>
<input type="hidden" name="extra" value="with-walk"/>
<button type="submit" class="btn btn-info">Discover</button>
</div>
</form>