Reduce macsuck bandwidth usage to database (#680)

* Reduce macsuck bandwidth usage to database

get_port_macs transfers a full list of all device_port.(mac,ip) in every macsuck.
With 8k devices and 40k interfaces it takes up around 15MB. Transferring them 8k times
during an 1h macsuck cycle requires bandwidth in the 300 to 400 mbit/s range.

This patch changes get_port_macs to be called inside walk_fwtable and only transfer
the macs found in the current target device/vlan.

lib/App/Netdisco/Util/PortMAC.pm

@@ -24,19 +24,18 @@ subroutines.
 =head2 get_port_macs
 
 Returns a Hash reference of C<< { MAC => IP } >> for all interface MAC
-addresses on all devices.
-
-If you need to filter for a given device, simply compare the IP (hash value)
-to your device's IP.
+addresses supplied as array reference
 
 =cut
 
 sub get_port_macs {
+
+    my ($fw_mac_list) = @_;
     my $port_macs = {};
 
     my $dp_macs
         = schema('netdisco')->resultset('DevicePort')
-        ->search( { mac => { '!=' => [ -and => (undef, '00:00:00:00:00:00') ] } },
+        ->search( { mac => { '-in' => $fw_mac_list } },
         { select => [ 'mac', 'ip' ],
           group_by => [ 'mac', 'ip' ] } );
     my $dp_cursor = $dp_macs->cursor;
@@ -46,7 +45,7 @@ sub get_port_macs {
 
     my $d_macs
         = schema('netdisco')->resultset('Device')
-        ->search( { mac => { '!=' => undef } },
+        ->search( { mac => { '-in' => $fw_mac_list } },
         { select => [ 'mac', 'ip' ] } );
     my $d_cursor = $d_macs->cursor;
     while ( my @vals = $d_cursor->next ) {

lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm

@@ -30,11 +30,11 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
   # cache the device ports to save hitting the database for many single rows
   my $device_ports = {map {($_->port => $_)}
                           $device->ports(undef, {prefetch => {neighbor_alias => 'device'}})->all};
-  my $port_macs = get_port_macs();
+
   my $interfaces = $snmp->interfaces;
 
   # get forwarding table data via basic snmp connection
-  my $fwtable = walk_fwtable($device, $interfaces, $port_macs, $device_ports);
+  my $fwtable = walk_fwtable($device, $interfaces, $device_ports);
 
   # ...then per-vlan if supported
   my @vlan_list = get_vlan_list($device);
@@ -43,7 +43,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
     foreach my $vlan (@vlan_list) {
       snmp_comm_reindex($snmp, $device, $vlan);
       my $pv_fwtable =
-        walk_fwtable($device, $interfaces, $port_macs, $device_ports, $vlan);
+        walk_fwtable($device, $interfaces, $device_ports, $vlan);
       $fwtable = {%$fwtable, %$pv_fwtable};
     }
   }
@@ -265,7 +265,7 @@ sub get_vlan_list {
 # walks the forwarding table (BRIDGE-MIB) for the device and returns a
 # table of node entries.
 sub walk_fwtable {
-  my ($device, $interfaces, $port_macs, $device_ports, $comm_vlan) = @_;
+  my ($device, $interfaces, $device_ports, $comm_vlan) = @_;
   my $skiplist = {}; # ports through which we can see another device
   my $cache = {};
 
@@ -278,6 +278,9 @@ sub walk_fwtable {
     ? {} : $snmp->qb_fw_vlan;
   my $bp_index = $snmp->bp_index;
 
+  my @fw_mac_list = values %$fw_mac;
+  my $port_macs = get_port_macs(\@fw_mac_list);
+
   # to map forwarding table port to device port we have
   #   fw_port -> bp_index -> interfaces