From 1d988bbf7cbcdf1501493154ea3e1ebbd166a86f Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Wed, 15 Mar 2023 14:44:42 +0000 Subject: [PATCH] implement ignore_layers, force_macsuck, force_arpnip config settings (#1002) * implementation of ignore_layers, force_macsuck, force_arpnip and macwalk * use new WalkJobs view to get devices needing macsuck * also new query for discoverall, arpwalk, nbtwalk * faux record has a last_defer stamp so we can see when the backend started * fix typo --- .../Netdisco/DB/Result/Virtual/WalkJobs.pm | 43 +++++++++++++++++++ lib/App/Netdisco/JobQueue/PostgreSQL.pm | 24 +++++------ lib/App/Netdisco/Util/Device.pm | 14 ++++-- .../Netdisco/Worker/Plugin/Arpnip/Nodes.pm | 1 - lib/App/Netdisco/Worker/Plugin/Arpwalk.pm | 14 +++--- .../Plugin/Discover/Neighbors/Routed.pm | 6 ++- .../Worker/Plugin/Discover/WithNodes.pm | 14 +++--- lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm | 15 ++++--- .../Netdisco/Worker/Plugin/Macsuck/Nodes.pm | 1 - lib/App/Netdisco/Worker/Plugin/Macwalk.pm | 14 +++--- lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm | 14 +++--- share/config.yml | 3 ++ 12 files changed, 112 insertions(+), 51 deletions(-) create mode 100644 lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm diff --git a/lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm b/lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm new file mode 100644 index 00000000..8affdcc6 --- /dev/null +++ b/lib/App/Netdisco/DB/Result/Virtual/WalkJobs.pm @@ -0,0 +1,43 @@ +package App::Netdisco::DB::Result::Virtual::WalkJobs; + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +__PACKAGE__->table_class('DBIx::Class::ResultSource::View'); + +__PACKAGE__->table('walk_jobs'); +__PACKAGE__->result_source_instance->is_virtual(1); +__PACKAGE__->result_source_instance->view_definition(< string_to_array(?, '') + OR (device_skip.deferrals >= ? + AND device_skip.last_defer > (LOCALTIMESTAMP - ? ::interval)))) + + WHERE admin.device IS NULL + AND device.ip IS NOT NULL + AND (device.vendor IS NULL OR device.vendor != 'netdisco') + + GROUP BY device.ip + HAVING count(device_skip.backend) < (SELECT count(distinct(backend)) FROM device_skip) + + ORDER BY device.ip ASC +ENDSQL +); + +__PACKAGE__->add_columns( + "ip", + { data_type => "inet", is_nullable => 0 }, +); + +__PACKAGE__->set_primary_key("ip"); + +1; diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index ec0c2581..dd88b820 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -72,6 +72,8 @@ sub jq_warm_thrusters { backend => setting('workers')->{'BACKEND'}, }, { for => 'update' }, )->update({ actionset => [] }); + # on backend restart, allow one retry of all devices which have + # reached max retry (max_deferrals) my $deferrals = setting('workers')->{'max_deferrals'} - 1; $rs->search({ backend => setting('workers')->{'BACKEND'}, @@ -80,7 +82,7 @@ sub jq_warm_thrusters { $rs->search({ backend => setting('workers')->{'BACKEND'}, - actionset => { -value => [] }, + actionset => { -value => [] }, # special syntax for matching empty ARRAY deferrals => 0, })->delete; @@ -89,19 +91,13 @@ sub jq_warm_thrusters { device => $_, actionset => $actionset{$_}, }, { key => 'primary' }) for keys %actionset; - }); - # fix up the pseudo devices which need layer 3 - # TODO remove this after next release - schema(vars->{'tenant'})->txn_do(sub { - my @hosts = grep { defined } - map { schema(vars->{'tenant'})->resultset('Device')->search_for_device($_->{only}) } - grep { exists $_->{only} and ref '' eq ref $_->{only} } - grep { exists $_->{driver} and $_->{driver} eq 'cli' } - @{ setting('device_auth') }; - - $_->update({ layers => \[q{overlay(layers placing '1' from 6 for 1)}] }) - for @hosts; + # add one faux record to allow *walk actions to see there is a backend running + $rs->update_or_create({ + backend => setting('workers')->{'BACKEND'}, + device => '255.255.255.255', + last_defer => \'LOCALTIMESTAMP', + }, { key => 'primary' }); }); } @@ -166,7 +162,7 @@ sub jq_getsome { },{ job => $job->id, -exists => $jobs->search({ - job => { '>' => $job->id }, + job => { '>' => $job->id }, status => { -like => 'queued-%' }, started => \[q/> (LOCALTIMESTAMP - ?::interval)/, setting('jobs_stale_after')], %job_properties, diff --git a/lib/App/Netdisco/Util/Device.pm b/lib/App/Netdisco/Util/Device.pm index 1489b633..472fc543 100644 --- a/lib/App/Netdisco/Util/Device.pm +++ b/lib/App/Netdisco/Util/Device.pm @@ -230,7 +230,8 @@ the local configuration to arpnip the device. The configuration items C and C are checked against the given IP. -Also checks if the device reports layer 3 capability. +Also checks if the device reports layer 3 capability, or matches +C or C. Returns false if the host is not permitted to arpnip the target device. @@ -241,7 +242,9 @@ sub is_arpnipable { my $device = get_device($ip) or return 0; return _bail_msg("is_arpnipable: $device has no layer 3 capability") - if ($device->in_storage() and not $device->has_layer(3)); + if ($device->in_storage() and not ($device->has_layer(3) + or check_acl_no($device, 'force_arpnip') + or check_acl_no($device, 'ignore_layers'))); return _bail_msg("is_arpnipable: $device matched arpnip_no") if check_acl_no($device, 'arpnip_no'); @@ -283,7 +286,8 @@ the local configuration to macsuck the device. The configuration items C and C are checked against the given IP. -Also checks if the device reports layer 2 capability. +Also checks if the device reports layer 2 capability, or matches +C or C. Returns false if the host is not permitted to macsuck the target device. @@ -294,7 +298,9 @@ sub is_macsuckable { my $device = get_device($ip) or return 0; return _bail_msg("is_macsuckable: $device has no layer 2 capability") - if ($device->in_storage() and not $device->has_layer(2)); + if ($device->in_storage() and not ($device->has_layer(2) + or check_acl_no($device, 'force_macsuck') + or check_acl_no($device, 'ignore_layers'))); return _bail_msg("is_macsuckable: $device matched macsuck_no") if check_acl_no($device, 'macsuck_no'); diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm index 8b789b99..34a4bb09 100644 --- a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm @@ -65,7 +65,6 @@ register_worker({ phase => 'store' }, sub { $device->ip, $v6; $device->update({last_arpnip => \$now}); - $device->update({layers => \[q{overlay(layers placing '1' from 6 for 1)}]}); my $status = $job->best_status; return Status->$status("Ended arpnip for $device"); diff --git a/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm index a7a21837..28e3a7b6 100644 --- a/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm @@ -4,16 +4,18 @@ use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use App::Netdisco::JobQueue 'jq_insert'; use Dancer::Plugin::DBIC 'schema'; register_worker({ phase => 'main' }, sub { my ($job, $workerconf) = @_; - my %queued = map {$_ => 1} jq_queued('arpnip'); - my @devices = schema('netdisco')->resultset('Device') - ->has_layer('3')->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; + my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs') + ->search(undef,{ bind => [ + 'arpnip', 'arpnip', + setting('workers')->{'max_deferrals'}, + setting('workers')->{'retry_after'}, + ]})->get_column('ip')->all; jq_insert([ map {{ @@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub { action => 'arpnip', username => $job->username, userip => $job->userip, - }} (@filtered_devices) + }} (@walk) ]); return Status->done('Queued arpnip job for all devices'); diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm index d8a65fa0..46f85d70 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm @@ -6,6 +6,7 @@ use App::Netdisco::Transport::SNMP; use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::Util::Device qw/get_device is_discoverable/; +use App::Netdisco::Util::Permission 'check_acl_no'; use App::Netdisco::JobQueue 'jq_insert'; register_worker({ phase => 'main', driver => 'snmp' }, sub { @@ -13,7 +14,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { return unless setting('discover_routed_neighbors'); my $device = $job->device; - return unless $device->in_storage and $device->has_layer(3); + return unless $device->in_storage and ($device->has_layer(3) + or check_acl_no($device, 'force_macsuck') + or check_acl_no($device, 'ignore_layers')); + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) or return Status->defer("discover failed: could not SNMP connect to $device"); diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm b/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm index 3333dfd8..b41b471a 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm @@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::JobQueue 'jq_insert'; -use Dancer::Plugin::DBIC 'schema'; +use App::Netdisco::Util::Permission 'check_acl_no'; register_worker({ phase => 'main' }, sub { my ($job, $workerconf) = @_; @@ -15,25 +15,29 @@ register_worker({ phase => 'main' }, sub { # arpniped/macsucked, queue those jobs now return unless $device->in_storage and $job->subaction eq 'with-nodes'; - if (!defined $device->last_macsuck and $device->has_layer(2)) { + if (!defined $device->last_macsuck and ($device->has_layer(2) + or check_acl_no($device, 'force_macsuck') + or check_acl_no($device, 'ignore_layers'))) { jq_insert({ device => $device->ip, action => 'macsuck', username => $job->username, userip => $job->userip, }); + debug sprintf ' [%s] queued macsuck', $device; } - if (!defined $device->last_arpnip and $device->has_layer(3)) { + if (!defined $device->last_arpnip and ($device->has_layer(3) + or check_acl_no($device, 'force_arpnip') + or check_acl_no($device, 'ignore_layers'))) { jq_insert({ device => $device->ip, action => 'arpnip', username => $job->username, userip => $job->userip, }); + debug sprintf ' [%s] queued arpnip', $device; } - - return Status->info("Queued macsuck and arpnip for $device."); }); true; diff --git a/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm b/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm index 3bace148..597f65d9 100644 --- a/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm +++ b/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm @@ -4,17 +4,18 @@ use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use App::Netdisco::JobQueue 'jq_insert'; use Dancer::Plugin::DBIC 'schema'; register_worker({ phase => 'main' }, sub { my ($job, $workerconf) = @_; - my %queued = map {$_ => 1} jq_queued('discover'); - my @devices = schema('netdisco')->resultset('Device')->search({ - -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], - })->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; + my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs') + ->search(undef,{ bind => [ + 'discover', 'discover', + setting('workers')->{'max_deferrals'}, + setting('workers')->{'retry_after'}, + ]})->get_column('ip')->all; jq_insert([ map {{ @@ -22,7 +23,7 @@ register_worker({ phase => 'main' }, sub { action => 'discover', username => $job->username, userip => $job->userip, - }} (@filtered_devices) + }} (@walk) ]); return Status->done('Queued discover job for all devices'); diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm index ebd2df1d..e59f5c40 100644 --- a/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm @@ -207,7 +207,6 @@ register_worker({ phase => 'store', $device->ip, $archived; $device->update({last_macsuck => \$now}); - $device->update({layers => \[q{overlay(layers placing '1' from 7 for 1)}]}); my $status = $job->best_status; return Status->$status("Ended macsuck for $device"); diff --git a/lib/App/Netdisco/Worker/Plugin/Macwalk.pm b/lib/App/Netdisco/Worker/Plugin/Macwalk.pm index 9bb96404..f6fd569c 100644 --- a/lib/App/Netdisco/Worker/Plugin/Macwalk.pm +++ b/lib/App/Netdisco/Worker/Plugin/Macwalk.pm @@ -4,16 +4,18 @@ use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use App::Netdisco::JobQueue 'jq_insert'; use Dancer::Plugin::DBIC 'schema'; register_worker({ phase => 'main' }, sub { my ($job, $workerconf) = @_; - my %queued = map {$_ => 1} jq_queued('macsuck'); - my @devices = schema('netdisco')->resultset('Device') - ->has_layer('2')->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; + my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs') + ->search(undef,{ bind => [ + 'macsuck', 'macsuck', + setting('workers')->{'max_deferrals'}, + setting('workers')->{'retry_after'}, + ]})->get_column('ip')->all; jq_insert([ map {{ @@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub { action => 'macsuck', username => $job->username, userip => $job->userip, - }} (@filtered_devices) + }} (@walk) ]); return Status->done('Queued macsuck job for all devices'); diff --git a/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm b/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm index f097260d..9ff3731a 100644 --- a/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm +++ b/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm @@ -4,16 +4,18 @@ use Dancer ':syntax'; use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use App::Netdisco::JobQueue 'jq_insert'; use Dancer::Plugin::DBIC 'schema'; register_worker({ phase => 'main' }, sub { my ($job, $workerconf) = @_; - my %queued = map {$_ => 1} jq_queued('nbtstat'); - my @devices = schema('netdisco')->resultset('Device') - ->has_layer('2')->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; + my @walk = schema(vars->{'tenant'})->resultset('Virtual::WalkJobs') + ->search(undef,{ bind => [ + 'macsuck', 'macsuck', + setting('workers')->{'max_deferrals'}, + setting('workers')->{'retry_after'}, + ]})->get_column('ip')->all; jq_insert([ map {{ @@ -21,7 +23,7 @@ register_worker({ phase => 'main' }, sub { action => 'nbtstat', username => $job->username, userip => $job->userip, - }} (@filtered_devices) + }} (@walk) ]); return Status->done('Queued nbtstat job for all devices'); diff --git a/share/config.yml b/share/config.yml index 4c272f04..d4c99ad4 100644 --- a/share/config.yml +++ b/share/config.yml @@ -351,6 +351,8 @@ discover_waps: true discover_phones: false discover_routed_neighbors: true discover_min_age: 0 +ignore_layers: [] +force_macsuck: [] macsuck_no: [] macsuck_only: [] macsuck_all_vlans: false @@ -373,6 +375,7 @@ macsuck_min_age: 0 snmpforce_v1: [] snmpforce_v2: [] snmpforce_v3: [] +force_arpnip: [] arpnip_no: [] arpnip_only: [] arpnip_min_age: 0