135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce macsuck bandwidth usage to database (#680)

Browse Source 
* Reduce macsuck bandwidth usage to database

get_port_macs transfers a full list of all device_port.(mac,ip) in every macsuck.
With 8k devices and 40k interfaces it takes up around 15MB. Transferring them 8k times
during an 1h macsuck cycle requires bandwidth in the 300 to 400 mbit/s range.

This patch changes get_port_macs to be called inside walk_fwtable and only transfer
the macs found in the current target device/vlan.
This commit is contained in:
Christian Ramseyer
2019-12-21 19:28:47 +01:00
committed by Oliver Gorwits
parent b211c7c7e2
commit 26d3fbdd40
2 changed files with 12 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lib/App/Netdisco/Util/PortMAC.pm
View File

@@ -24,19 +24,18 @@ subroutines.
=head2 get_port_macs =head2 get_port_macs
Returns a Hash reference of C<< { MAC => IP } >> for all interface MAC Returns a Hash reference of C<< { MAC => IP } >> for all interface MAC
addresses on all devices. addresses supplied as array reference
If you need to filter for a given device, simply compare the IP (hash value)
to your device's IP.
=cut =cut
sub get_port_macs { sub get_port_macs {
my ($fw_mac_list) = @_;
my $port_macs = {}; my $port_macs = {};
my $dp_macs my $dp_macs
= schema('netdisco')->resultset('DevicePort') = schema('netdisco')->resultset('DevicePort')
->search( { mac => { '!=' => [ -and => (undef, '00:00:00:00:00:00') ] } }, ->search( { mac => { '-in' => $fw_mac_list } },
{ select => [ 'mac', 'ip' ], { select => [ 'mac', 'ip' ],
group_by => [ 'mac', 'ip' ] } ); group_by => [ 'mac', 'ip' ] } );
my $dp_cursor = $dp_macs->cursor; my $dp_cursor = $dp_macs->cursor;
@@ -46,7 +45,7 @@ sub get_port_macs {
my $d_macs my $d_macs
= schema('netdisco')->resultset('Device') = schema('netdisco')->resultset('Device')
->search( { mac => { '!=' => undef } }, ->search( { mac => { '-in' => $fw_mac_list } },
{ select => [ 'mac', 'ip' ] } ); { select => [ 'mac', 'ip' ] } );
my $d_cursor = $d_macs->cursor; my $d_cursor = $d_macs->cursor;
while ( my @vals = $d_cursor->next ) { while ( my @vals = $d_cursor->next ) {

11
lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm
View File

@@ -30,11 +30,11 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
# cache the device ports to save hitting the database for many single rows # cache the device ports to save hitting the database for many single rows
my $device_ports = {map {($_->port => $_)} my $device_ports = {map {($_->port => $_)}
$device->ports(undef, {prefetch => {neighbor_alias => 'device'}})->all}; $device->ports(undef, {prefetch => {neighbor_alias => 'device'}})->all};
my $port_macs = get_port_macs();
my $interfaces = $snmp->interfaces; my $interfaces = $snmp->interfaces;
# get forwarding table data via basic snmp connection # get forwarding table data via basic snmp connection
my $fwtable = walk_fwtable($device, $interfaces, $port_macs, $device_ports); my $fwtable = walk_fwtable($device, $interfaces, $device_ports);
# ...then per-vlan if supported # ...then per-vlan if supported
my @vlan_list = get_vlan_list($device); my @vlan_list = get_vlan_list($device);
@@ -43,7 +43,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
foreach my $vlan (@vlan_list) { foreach my $vlan (@vlan_list) {
snmp_comm_reindex($snmp, $device, $vlan); snmp_comm_reindex($snmp, $device, $vlan);
my $pv_fwtable = my $pv_fwtable =
walk_fwtable($device, $interfaces, $port_macs, $device_ports, $vlan); walk_fwtable($device, $interfaces, $device_ports, $vlan);
$fwtable = {%$fwtable, %$pv_fwtable}; $fwtable = {%$fwtable, %$pv_fwtable};
} }
} }
@@ -265,7 +265,7 @@ sub get_vlan_list {
# walks the forwarding table (BRIDGE-MIB) for the device and returns a # walks the forwarding table (BRIDGE-MIB) for the device and returns a
# table of node entries. # table of node entries.
sub walk_fwtable { sub walk_fwtable {
my ($device, $interfaces, $port_macs, $device_ports, $comm_vlan) = @_; my ($device, $interfaces, $device_ports, $comm_vlan) = @_;
my $skiplist = {}; # ports through which we can see another device my $skiplist = {}; # ports through which we can see another device
my $cache = {}; my $cache = {};
@@ -278,6 +278,9 @@ sub walk_fwtable {
? {} : $snmp->qb_fw_vlan; ? {} : $snmp->qb_fw_vlan;
my $bp_index = $snmp->bp_index; my $bp_index = $snmp->bp_index;
my @fw_mac_list = values %$fw_mac;
my $port_macs = get_port_macs(\@fw_mac_list);
# to map forwarding table port to device port we have # to map forwarding table port to device port we have
# fw_port -> bp_index -> interfaces # fw_port -> bp_index -> interfaces