135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement ignore_layers, force_macsuck, force_arpnip config settings (#1002)

Browse Source 
* implementation of ignore_layers, force_macsuck, force_arpnip and macwalk

* use new WalkJobs view to get devices needing macsuck

* also new query for discoverall, arpwalk, nbtwalk

* faux record has a last_defer stamp so we can see when the backend started

* fix typo
This commit is contained in:
Oliver Gorwits
2023-03-15 14:44:42 +00:00
committed by GitHub
parent 062895df10
commit 1d988bbf7c
12 changed files with 112 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm Normal file
View File

@@ -0,0 +1,43 @@
package App::Netdisco::DB::Result::Virtual::WalkJobs;
use strict;
use warnings;
use base 'DBIx::Class::Core';
__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
__PACKAGE__->table('walk_jobs');
__PACKAGE__->result_source_instance->is_virtual(1);
__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
SELECT ip
FROM device
LEFT OUTER JOIN admin ON (device.ip = admin.device
AND admin.status = 'queued'
AND admin.action = ?)
FULL OUTER JOIN device_skip ON (device_skip.device = device.ip
AND (device_skip.actionset @> string_to_array(?, '')
OR (device_skip.deferrals >= ?
AND device_skip.last_defer > (LOCALTIMESTAMP - ? ::interval))))
WHERE admin.device IS NULL
AND device.ip IS NOT NULL
AND (device.vendor IS NULL OR device.vendor != 'netdisco')
GROUP BY device.ip
HAVING count(device_skip.backend) < (SELECT count(distinct(backend)) FROM device_skip)
ORDER BY device.ip ASC
ENDSQL
);
__PACKAGE__->add_columns(
"ip",
{ data_type => "inet", is_nullable => 0 },
);
__PACKAGE__->set_primary_key("ip");
1;

24
lib/App/Netdisco/JobQueue/PostgreSQL.pm
View File

@@ -72,6 +72,8 @@ sub jq_warm_thrusters {
backend => setting('workers')->{'BACKEND'},
}, { for => 'update' }, )->update({ actionset => [] });
# on backend restart, allow one retry of all devices which have
# reached max retry (max_deferrals)
my $deferrals = setting('workers')->{'max_deferrals'} - 1;
$rs->search({
backend => setting('workers')->{'BACKEND'},
@@ -80,7 +82,7 @@ sub jq_warm_thrusters {
$rs->search({
backend => setting('workers')->{'BACKEND'},
actionset => { -value => [] },
actionset => { -value => [] }, # special syntax for matching empty ARRAY
deferrals => 0,
})->delete;
@@ -89,19 +91,13 @@ sub jq_warm_thrusters {
device => $_,
actionset => $actionset{$_},
}, { key => 'primary' }) for keys %actionset;
});
# fix up the pseudo devices which need layer 3
# TODO remove this after next release
schema(vars->{'tenant'})->txn_do(sub {
my @hosts = grep { defined }
map { schema(vars->{'tenant'})->resultset('Device')->search_for_device($_->{only}) }
grep { exists $_->{only} and ref '' eq ref $_->{only} }
grep { exists $_->{driver} and $_->{driver} eq 'cli' }
@{ setting('device_auth') };
$_->update({ layers => \[q{overlay(layers placing '1' from 6 for 1)}] })
for @hosts;
# add one faux record to allow *walk actions to see there is a backend running
$rs->update_or_create({
backend => setting('workers')->{'BACKEND'},
device => '255.255.255.255',
last_defer => \'LOCALTIMESTAMP',
}, { key => 'primary' });
});
}
@@ -166,7 +162,7 @@ sub jq_getsome {
},{
job => $job->id,
-exists => $jobs->search({
job => { '>' => $job->id },
job => { '>' => $job->id },
status => { -like => 'queued-%' },
started => \[q/> (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')],
%job_properties,

14
lib/App/Netdisco/Util/Device.pm
View File

@@ -230,7 +230,8 @@ the local configuration to arpnip the device.
The configuration items C<arpnip_no> and C<arpnip_only> are checked
against the given IP.
Also checks if the device reports layer 3 capability.
Also checks if the device reports layer 3 capability, or matches
C<force_arpnip> or C<ignore_layers>.
Returns false if the host is not permitted to arpnip the target device.
@@ -241,7 +242,9 @@ sub is_arpnipable {
my $device = get_device($ip) or return 0;
return _bail_msg("is_arpnipable: $device has no layer 3 capability")
if ($device->in_storage() and not $device->has_layer(3));
if ($device->in_storage() and not ($device->has_layer(3)
or check_acl_no($device, 'force_arpnip')
or check_acl_no($device, 'ignore_layers')));
return _bail_msg("is_arpnipable: $device matched arpnip_no")
if check_acl_no($device, 'arpnip_no');
@@ -283,7 +286,8 @@ the local configuration to macsuck the device.
The configuration items C<macsuck_no> and C<macsuck_only> are checked
against the given IP.
Also checks if the device reports layer 2 capability.
Also checks if the device reports layer 2 capability, or matches
C<force_macsuck> or C<ignore_layers>.
Returns false if the host is not permitted to macsuck the target device.
@@ -294,7 +298,9 @@ sub is_macsuckable {
my $device = get_device($ip) or return 0;
return _bail_msg("is_macsuckable: $device has no layer 2 capability")
if ($device->in_storage() and not $device->has_layer(2));
if ($device->in_storage() and not ($device->has_layer(2)
or check_acl_no($device, 'force_macsuck')
or check_acl_no($device, 'ignore_layers')));
return _bail_msg("is_macsuckable: $device matched macsuck_no")
if check_acl_no($device, 'macsuck_no');

1
lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
View File

@@ -65,7 +65,6 @@ register_worker({ phase => 'store' }, sub {
$device->ip, $v6;
$device->update({last_arpnip => \$now});
$device->update({layers => \[q{overlay(layers placing '1' from 6 for 1)}]});
my $status = $job->best_status;
return Status->$status("Ended arpnip for $device");

14
lib/App/Netdisco/Worker/Plugin/Arpwalk.pm
View File

@@ -4,16 +4,18 @@ use Dancer ':syntax';
use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
use App::Netdisco::JobQueue 'jq_insert';
use Dancer::Plugin::DBIC 'schema';
register_worker({ phase => 'main' }, sub {
my ($job, $workerconf) = @_;
my %queued = map {$_ => 1} jq_queued('arpnip');
my @devices = schema('netdisco')->resultset('Device')
->has_layer('3')->get_column('ip')->all;
my @filtered_devices = grep {!exists $queued{$_}} @devices;
my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
->search(undef,{ bind => [
'arpnip', 'arpnip',
setting('workers')->{'max_deferrals'},
setting('workers')->{'retry_after'},
]})->get_column('ip')->all;
jq_insert([
map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
action => 'arpnip',
username => $job->username,
userip => $job->userip,
}} (@filtered_devices)
}} (@walk)
]);
return Status->done('Queued arpnip job for all devices');

6
lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm
View File

@@ -6,6 +6,7 @@ use App::Netdisco::Transport::SNMP;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::Util::Device qw/get_device is_discoverable/;
use App::Netdisco::Util::Permission 'check_acl_no';
use App::Netdisco::JobQueue 'jq_insert';
register_worker({ phase => 'main', driver => 'snmp' }, sub {
@@ -13,7 +14,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
return unless setting('discover_routed_neighbors');
my $device = $job->device;
return unless $device->in_storage and $device->has_layer(3);
return unless $device->in_storage and ($device->has_layer(3)
or check_acl_no($device, 'force_macsuck')
or check_acl_no($device, 'ignore_layers'));
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("discover failed: could not SNMP connect to $device");

14
lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm
View File

@@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::JobQueue 'jq_insert';
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Util::Permission 'check_acl_no';
register_worker({ phase => 'main' }, sub {
my ($job, $workerconf) = @_;
@@ -15,25 +15,29 @@ register_worker({ phase => 'main' }, sub {
# arpniped/macsucked, queue those jobs now
return unless $device->in_storage and $job->subaction eq 'with-nodes';
if (!defined $device->last_macsuck and $device->has_layer(2)) {
if (!defined $device->last_macsuck and ($device->has_layer(2)
or check_acl_no($device, 'force_macsuck')
or check_acl_no($device, 'ignore_layers'))) {
jq_insert({
device => $device->ip,
action => 'macsuck',
username => $job->username,
userip => $job->userip,
});
debug sprintf ' [%s] queued macsuck', $device;
}
if (!defined $device->last_arpnip and $device->has_layer(3)) {
if (!defined $device->last_arpnip and ($device->has_layer(3)
or check_acl_no($device, 'force_arpnip')
or check_acl_no($device, 'ignore_layers'))) {
jq_insert({
device => $device->ip,
action => 'arpnip',
username => $job->username,
userip => $job->userip,
});
debug sprintf ' [%s] queued arpnip', $device;
}
return Status->info("Queued macsuck and arpnip for $device.");
});
true;

15
lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm
View File

@@ -4,17 +4,18 @@ use Dancer ':syntax';
use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
use App::Netdisco::JobQueue 'jq_insert';
use Dancer::Plugin::DBIC 'schema';
register_worker({ phase => 'main' }, sub {
my ($job, $workerconf) = @_;
my %queued = map {$_ => 1} jq_queued('discover');
my @devices = schema('netdisco')->resultset('Device')->search({
-or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
})->get_column('ip')->all;
my @filtered_devices = grep {!exists $queued{$_}} @devices;
my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
->search(undef,{ bind => [
'discover', 'discover',
setting('workers')->{'max_deferrals'},
setting('workers')->{'retry_after'},
]})->get_column('ip')->all;
jq_insert([
map {{
@@ -22,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
action => 'discover',
username => $job->username,
userip => $job->userip,
}} (@filtered_devices)
}} (@walk)
]);
return Status->done('Queued discover job for all devices');

1
lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm
View File

@@ -207,7 +207,6 @@ register_worker({ phase => 'store',
$device->ip, $archived;
$device->update({last_macsuck => \$now});
$device->update({layers => \[q{overlay(layers placing '1' from 7 for 1)}]});
my $status = $job->best_status;
return Status->$status("Ended macsuck for $device");

14
lib/App/Netdisco/Worker/Plugin/Macwalk.pm
View File

@@ -4,16 +4,18 @@ use Dancer ':syntax';
use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
use App::Netdisco::JobQueue 'jq_insert';
use Dancer::Plugin::DBIC 'schema';
register_worker({ phase => 'main' }, sub {
my ($job, $workerconf) = @_;
my %queued = map {$_ => 1} jq_queued('macsuck');
my @devices = schema('netdisco')->resultset('Device')
->has_layer('2')->get_column('ip')->all;
my @filtered_devices = grep {!exists $queued{$_}} @devices;
my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
->search(undef,{ bind => [
'macsuck', 'macsuck',
setting('workers')->{'max_deferrals'},
setting('workers')->{'retry_after'},
]})->get_column('ip')->all;
jq_insert([
map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
action => 'macsuck',
username => $job->username,
userip => $job->userip,
}} (@filtered_devices)
}} (@walk)
]);
return Status->done('Queued macsuck job for all devices');

14
lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm
View File

@@ -4,16 +4,18 @@ use Dancer ':syntax';
use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
use App::Netdisco::JobQueue 'jq_insert';
use Dancer::Plugin::DBIC 'schema';
register_worker({ phase => 'main' }, sub {
my ($job, $workerconf) = @_;
my %queued = map {$_ => 1} jq_queued('nbtstat');
my @devices = schema('netdisco')->resultset('Device')
->has_layer('2')->get_column('ip')->all;
my @filtered_devices = grep {!exists $queued{$_}} @devices;
my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs')
->search(undef,{ bind => [
'macsuck', 'macsuck',
setting('workers')->{'max_deferrals'},
setting('workers')->{'retry_after'},
]})->get_column('ip')->all;
jq_insert([
map {{
@@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub {
action => 'nbtstat',
username => $job->username,
userip => $job->userip,
}} (@filtered_devices)
}} (@walk)
]);
return Status->done('Queued nbtstat job for all devices');