135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#925 implement ignore_deviceports and hide_deviceports

Browse Source
This commit is contained in:
Oliver Gorwits
2022-09-20 20:32:35 +01:00
parent 7d7d052bb6
commit 790c51b257
12 changed files with 284 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
lib/App/Netdisco/Configuration.pm
View File

@@ -204,6 +204,30 @@ if (ref {} eq ref setting('macsuck_no_deviceport')) {
}
else { config->{'macsuck_no_deviceport'} ||= [] }
if (ref {} eq ref setting('hide_deviceports')) {
config->{'hide_deviceports'} = [ setting('hide_deviceports') ];
}
else { config->{'hide_deviceports'} ||= [] }
if (ref {} eq ref setting('ignore_deviceports')) {
config->{'ignore_deviceports'} = [ setting('ignore_deviceports') ];
}
else { config->{'ignore_deviceports'} ||= [] }
# copy old ignore_* into new settings
if (scalar @{ config->{'ignore_interfaces'} }) {
config->{'host_groups'}->{'__IGNORE_INTERFACES__'}
= config->{'ignore_interfaces'};
}
if (scalar @{ config->{'ignore_interface_types'} }) {
config->{'host_groups'}->{'__IGNORE_INTERFACE_TYPES__'}
= config->{'ignore_interface_types'};
}
if (scalar @{ config->{'ignore_notpresent_types'} }) {
config->{'host_groups'}->{'__NOTPRESENT_TYPES__'}
= config->{'ignore_notpresent_types'};
}
# copy devices_no and devices_only into others
foreach my $name (qw/devices_no devices_only
discover_no macsuck_no arpnip_no nbtstat_no

38
lib/App/Netdisco/DB/Result/DeviceIp.pm
View File

@@ -6,6 +6,8 @@ use strict;
use warnings;
use base 'App::Netdisco::DB::Result';
use Sub::Install;
__PACKAGE__->table("device_ip");
__PACKAGE__->add_columns(
"ip",
@@ -47,9 +49,41 @@ routed port or virtual interface).
=cut
__PACKAGE__->add_unique_constraint(['alias']);
__PACKAGE__->belongs_to( device_port => 'App::Netdisco::DB::Result::DevicePort',
{ 'foreign.port' => 'self.port', 'foreign.ip' => 'self.ip' } );
=head2 device_port fields
All C<device_port> fields are mapped to accessors on this object.
=cut
foreach my $field (qw/
descr
up
up_admin
type
duplex
duplex_admin
speed
speed_admin
name
mac
mtu
stp
remote_ip
remote_port
remote_type
remote_id
vlan
pvid
lastchange
/) {
Sub::Install::install_sub({
code => sub { return eval { (shift)->device_port->$field } },
as => $field,
});
}
1;

2
lib/App/Netdisco/Transport/SNMP.pm
View File

@@ -96,7 +96,7 @@ sub test_connection {
my $addr = NetAddr::IP::Lite->new($ip) or return undef;
# avoid renumbering to localhost loopbacks
return undef if $addr->addr eq '0.0.0.0'
or check_acl_no($addr->addr, 'group:__LOCAL_ADDRESSES__');
or check_acl_no($addr->addr, 'group:__LOOPBACK_ADDRESSES__');
my $device = schema('netdisco')->resultset('Device')
->new_result({ ip => $addr->addr }) or return undef;
my $readers = $class->instance->readers or return undef;

45
lib/App/Netdisco/Util/Permission.pm
View File

@@ -94,25 +94,26 @@ sub check_acl {
my $real_ip = $thing;
if (blessed $thing) {
$real_ip = ($thing->can('alias') ? $thing->alias : (
$thing->can('ip') ? $thing->ip : (
$thing->can('addr') ? $thing->addr : $thing )));
$real_ip = (
$thing->can('alias') ? $thing->alias : (
$thing->can('ip') ? $thing->ip : (
$thing->can('addr') ? $thing->addr : $thing )));
}
return 0 if !defined $real_ip
or blessed $real_ip; # class we do not understand
return 0 if blessed $real_ip; # class we do not understand
$real_ip ||= ''; # valid to be empty
$config = [$config] if ref '' eq ref $config;
$config = [$config] if ref q{} eq ref $config;
if (ref [] ne ref $config) {
error "error: acl is not a single item or list (cannot compare to $real_ip)";
error "error: acl is not a single item or list (cannot compare to '$real_ip')";
return 0;
}
my $all = (scalar grep {$_ eq 'op:and'} @$config);
# common case of using plain IP in ACL, so string compare for speed
my $find = (scalar grep {not reftype $_ and $_ eq $real_ip} @$config);
return 1 if $find and not $all;
return 1 if $real_ip and $find and not $all;
my $addr = NetAddr::IP::Lite->new($real_ip) or return 0;
my $addr = NetAddr::IP::Lite->new($real_ip);
my $name = undef; # only look up once, and only if qr// is used
my $ropt = { retry => 1, retrans => 1, udp_timeout => 1, tcp_timeout => 2 };
my $qref = ref qr//;
@@ -122,6 +123,9 @@ sub check_acl {
next INLIST if !defined $item or $item eq 'op:and';
if ($qref eq ref $item) {
# if no IP addr, cannot match its dns
next INLIST unless $addr;
$name = ($name || hostname_from_ip($addr->addr, $ropt) || '!!none!!');
if ($name =~ $item) {
return 1 if not $all;
@@ -147,16 +151,23 @@ sub check_acl {
next INLIST;
}
if ($item =~ m/^([^:]+):([^:]+)$/) {
if ($item =~ m/^([^:]+):([^:]*)$/) {
my $prop = $1;
my $match = $2;
my $match = $2 || '';
# if not an object, we can't do much with properties
next INLIST unless blessed $thing;
# lazy version of vendor: and model:
if ($neg xor ($thing->can($prop) and defined eval { $thing->$prop }
and $thing->$prop =~ m/^$match$/)) {
# prop:val
if ($neg xor ($thing->can($prop) and
defined eval { $thing->$prop } and
ref $thing->$prop eq q{}
and $thing->$prop =~ m/^$match$/) ) {
return 1 if not $all;
}
# empty or missing property
elsif ($neg xor ($match eq q{} and
(!defined eval { $thing->$prop } or $thing->$prop eq q{})) ) {
return 1 if not $all;
}
else {
@@ -169,6 +180,9 @@ sub check_acl {
my $first = $1;
my $last = $2;
# if no IP addr, cannot match IP range
next INLIST unless $addr;
if ($item =~ m/:/) {
next INLIST if $addr->bits != 128 and not $all;
@@ -208,6 +222,9 @@ sub check_acl {
# could be something in error, and IP/host is only option left
next INLIST if ref $item;
# if no IP addr, cannot match IP prefix
next INLIST unless $addr;
my $ip = NetAddr::IP::Lite->new($item)
or next INLIST;
next INLIST if $ip->bits != $addr->bits and not $all;

30
lib/App/Netdisco/Web/Plugin/Device/Ports.pm
View File

@@ -4,6 +4,7 @@ use Dancer ':syntax';
use Dancer::Plugin::DBIC;
use Dancer::Plugin::Auth::Extensible;
use App::Netdisco::Util::Permission 'check_acl_no';
use App::Netdisco::Util::Port 'port_reconfig_check';
use App::Netdisco::Util::Web (); # for sort_port
use App::Netdisco::Web::Plugin;
@@ -214,6 +215,35 @@ get '/ajax/content/device/ports' => require_login sub {
}
}
# filter out hidden ones
if (not param('p_include_hidden')) {
my $device_ips = {};
map { push @{ $device_ips->{$_->port} }, $_ }
$device->device_ips(undef, {prefetch => 'device_port'})->all;
map { push @{ $device_ips->{$_->port} }, $_ }
grep { ! exists $device_ips->{$_->port} }
@results;
foreach my $map (@{ setting('hide_deviceports')}) {
next unless ref {} eq ref $map;
foreach my $key (sort keys %$map) {
# lhs matches device, rhs matches port
next unless check_acl_no($device, $key);
PORT: foreach my $port (sort keys %$device_ips) {
foreach my $thing (@{ $device_ips->{$port} }) {
next unless check_acl_no($thing, $map->{$key});
@results = grep { $_->port ne $port } @results;
next PORT;
}
}
}
}
}
# sort ports
@results = sort { &App::Netdisco::Util::Web::sort_port($a->port, $b->port) } @results;

2
lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm
View File

@@ -42,7 +42,7 @@ sub gather_subnets {
my $addr = $ip->addr;
next if $addr eq '0.0.0.0';
next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__');
next if check_acl_no($ip, 'group:__LOOPBACK_ADDRESSES__');
next if setting('ignore_private_nets') and $ip->is_rfc1918;
my $netmask = $ip_netmask->{$addr} || $ip->bits();

6
lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm
View File

@@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::Transport::SNMP ();
use App::Netdisco::Util::Permission 'check_acl_only';
use App::Netdisco::Util::Permission 'check_acl_no';
use App::Netdisco::Util::DNS 'ipv4_from_hostname';
use App::Netdisco::Util::Device 'is_discoverable';
use Dancer::Plugin::DBIC 'schema';
@@ -44,8 +44,8 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
foreach my $key (sort keys %$map) {
# lhs matches device, rhs matches device_ip
if (check_acl_only($device, $key)
and check_acl_only($alias, $map->{$key})) {
if (check_acl_no($device, $key)
and check_acl_no($alias, $map->{$key})) {
if (not is_discoverable( $alias->alias )) {
debug sprintf ' [%s] device - cannot renumber to %s - not discoverable',

2
lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm
View File

@@ -185,7 +185,7 @@ sub store_neighbors {
# useable remote IP...
if ((! $r_netaddr) or ($remote_ip eq '0.0.0.0') or
check_acl_no($remote_ip, 'group:__LOCAL_ADDRESSES__')) {
check_acl_no($remote_ip, 'group:__LOOPBACK_ADDRESSES__')) {
if ($remote_id) {
my $devices = schema('netdisco')->resultset('Device');

116
lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm
View File

@@ -5,7 +5,7 @@ use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::Transport::SNMP ();
use App::Netdisco::Util::Permission qw/check_acl_no check_acl_only/;
use App::Netdisco::Util::Permission 'check_acl_no';
use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
use App::Netdisco::Util::Device 'get_device';
use App::Netdisco::Util::DNS 'hostname_from_ip';
@@ -66,7 +66,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
my $protect = setting('snmp_field_protection')->{'device'} || {};
my %dirty = $device->get_dirty_columns;
foreach my $field (keys %dirty) {
next unless check_acl_only($ip, $protect->{$field});
next unless check_acl_no($ip, $protect->{$field});
if (!defined $dirty{$field} or $dirty{$field} eq '') {
return $job->cancel("discover cancelled: $ip failed to return valid $field");
}
@@ -101,7 +101,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
});
}
return Status->info(" [$device] device - OK to continue discover");
return Status->info(" [$device] device - OK to continue discover (not a duplicate)");
});
register_worker({ phase => 'early', driver => 'snmp' }, sub {
@@ -113,7 +113,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("discover failed: could not SNMP connect to $device");
my $pass = Status->info(" [$device] device - OK to continue discover");
my $pass = Status->info(" [$device] device - OK to continue discover (valid interfaces)");
my $interfaces = $snmp->interfaces;
# OK if no interfaces
@@ -172,6 +172,8 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
});
});
# NOTE must come after the IP Aliases gathering for ignore ACLs to work
register_worker({ phase => 'early', driver => 'snmp' }, sub {
my ($job, $workerconf) = @_;
@@ -180,6 +182,13 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("discover failed: could not SNMP connect to $device");
# gather device_ips for use in ACLs later
my $device_ips = {};
foreach my $dip ($device->device_ips()->all) {
next unless defined $dip->port and $dip->port;
push @{ $device_ips->{ $dip->port } }, $dip;
}
my $interfaces = $snmp->interfaces;
my $i_type = $snmp->i_type;
my $i_ignore = $snmp->i_ignore;
@@ -222,41 +231,20 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
}
# build device interfaces suitable for DBIC
my %interfaces;
foreach my $entry (keys %$interfaces) {
my %deviceports;
PORT: foreach my $entry (keys %$interfaces) {
my $port = $interfaces->{$entry};
if (not $port) {
debug sprintf ' [%s] interfaces - ignoring %s (no port mapping)',
$device->ip, $entry;
next;
}
if (scalar grep {$port =~ m/^$_$/} @{setting('ignore_interfaces') || []}) {
debug sprintf
' [%s] interfaces - ignoring %s (%s) (config:ignore_interfaces)',
$device->ip, $entry, $port;
next;
next PORT;
}
if (exists $i_ignore->{$entry}) {
debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s) (SNMP::Info::i_ignore)',
$device->ip, $entry, $port, ($i_type->{$entry} || '');
next;
}
# Skip interfaces by type filter
if (defined $i_type->{$entry} and (scalar grep {$i_type->{$entry} =~ m/^$_$/} @{setting('ignore_interface_types') || []})) {
debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s) (config:ignore_interface_types)',
$device->ip, $entry, $port, $i_type->{$entry};
next;
}
# Skip interfaces which are 'notPresent' and match the notpresent type filter
if (defined $i_up->{$entry} and defined $i_type->{$entry} and $i_up->{$entry} eq 'notPresent' and (scalar grep {$i_type->{$entry} =~ m/^$_$/} @{setting('ignore_notpresent_types') || []}) ) {
debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s) (config:ignore_notpresent_types)',
$device->ip, $entry, $port, $i_up->{$entry};
next;
next PORT;
}
my $lc = $i_lastchange->{$entry} || 0;
@@ -285,7 +273,9 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
}
}
$interfaces{$port} = {
# create a DBIx::Class row for this port which can be used to test ACLs
# also include the Device IP alias if we have one for L3 interfaces
$deviceports{$port} = {
port => $port,
descr => $i_descr->{$entry},
up => $i_up->{$entry},
@@ -304,19 +294,57 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
has_subinterfaces => 'false',
is_master => 'false',
slave_of => undef,
lastchange => $lc,
lastchange => $lc,
};
}
# must do this after building %interfaces so that we can set is_master
if (scalar @{ setting('ignore_deviceports') }) {
foreach my $port (keys %$device_ips) {
if (!exists $deviceports{$port}) {
delete $device_ips->{$port};
next;
}
foreach my $dip (@{ $device_ips->{$port} }) {
$dip->set_inflated_columns({ device_port => $deviceports{$port} });
}
}
foreach my $port (keys %deviceports) {
next if exists $device_ips->{$port};
push @{ $device_ips->{$port} },
schema('netdisco')->resultset('DevicePort')
->new_result( $deviceports{$port} );
}
foreach my $map (@{ setting('ignore_deviceports')}) {
next unless ref {} eq ref $map;
foreach my $key (sort keys %$map) {
# lhs matches device, rhs matches port
next unless check_acl_no($device, $key);
PORT: foreach my $port (sort keys %$device_ips) {
foreach my $thing (@{ $device_ips->{$port} }) {
next unless check_acl_no($thing, $map->{$key});
debug sprintf ' [%s] interfaces - ignoring %s (config:ignore_deviceports)',
$device->ip, $port;
delete $deviceports{$port};
next PORT;
}
}
}
}
}
# must do this after building %deviceports so that we can set is_master
foreach my $sidx (keys %$agg_ports) {
my $slave = $interfaces->{$sidx} or next;
next unless defined $agg_ports->{$sidx}; # slave without a master?!
my $master = $interfaces->{ $agg_ports->{$sidx} } or next;
next unless exists $interfaces{$slave} and exists $interfaces{$master};
next unless exists $deviceports{$slave} and exists $deviceports{$master};
$interfaces{$slave}->{slave_of} = $master;
$interfaces{$master}->{is_master} = 'true';
$deviceports{$slave}->{slave_of} = $master;
$deviceports{$master}->{is_master} = 'true';
}
# also for VLAN subinterfaces
@@ -325,27 +353,29 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
# parent without subinterfaces?
next unless defined $i_subs->{$pidx}
and ref [] eq ref $i_subs->{$pidx}
and scalar @{ $i_subs->{$pidx} };
and scalar @{ $i_subs->{$pidx} }
and exists $deviceports{$parent};
$interfaces{$parent}->{has_subinterfaces} = 'true';
$deviceports{$parent}->{has_subinterfaces} = 'true';
foreach my $sidx (@{ $i_subs->{$pidx} }) {
my $sub = $interfaces->{$sidx} or next;
$interfaces{$sub}->{slave_of} = $parent;
next unless exists $deviceports{$sub};
$deviceports{$sub}->{slave_of} = $parent;
}
}
# support for Hooks
vars->{'hook_data'}->{'ports'} = [values %interfaces];
vars->{'hook_data'}->{'ports'} = [values %deviceports];
schema('netdisco')->resultset('DevicePort')->txn_do_locked(sub {
my $gone = $device->ports->delete({keep_nodes => 1});
debug sprintf ' [%s] interfaces - removed %d interfaces',
$device->ip, $gone;
$device->update_or_insert(undef, {for => 'update'});
$device->ports->populate([values %interfaces]);
$device->ports->populate([values %deviceports]);
return Status->info(sprintf ' [%s] interfaces - added %d new interfaces',
$device->ip, scalar values %interfaces);
$device->ip, scalar values %deviceports);
});
});
@@ -397,7 +427,7 @@ sub _get_ipv4_aliases {
my $addr = $ip->addr;
next if $addr eq '0.0.0.0';
next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__');
next if check_acl_no($ip, 'group:__LOOPBACK_ADDRESSES__');
next if setting('ignore_private_nets') and $ip->is_rfc1918;
my $iid = $ip_index->{$addr};
@@ -450,7 +480,7 @@ sub _get_ipv6_aliases {
my $addr = $ip->addr;
next if $addr eq '::0';
next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__');
next if check_acl_no($ip, 'group:__LOOPBACK_ADDRESSES__');
my $port = $interfaces->{ $ipv6_index->{$iid} };
my $subnet = $ipv6_pfxlen->{$iid}