move macsuck and macwalk to worker plugins (macsuck needs snmp scope guard)

lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm

@@ -1,18 +0,0 @@
-package App::Netdisco::Backend::Worker::Poller::Macsuck;
-
-use App::Netdisco::Core::Macsuck 'do_macsuck';
-use App::Netdisco::Util::Device 'is_macsuckable_now';
-
-use Role::Tiny;
-use namespace::clean;
-
-with 'App::Netdisco::Backend::Worker::Poller::Common';
-
-sub macsuck_action { \&do_macsuck }
-sub macsuck_filter { \&is_macsuckable_now }
-sub macsuck_layer { 2 }
-
-sub macwalk { (shift)->_walk_body('macsuck', @_) }
-sub macsuck { (shift)->_single_body('macsuck', @_) }
-
-1;

lib/App/Netdisco/Worker/Plugin/Macsuck.pm

@@ -0,0 +1,38 @@
+package App::Netdisco::Worker::Plugin::Macsuck;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::Util::Device 'is_macsuckable_now';
+use App::Netdisco::Transport::SNMP ();
+
+register_worker({ primary => true }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  return Status->error('macsuck failed: unable to interpret device param')
+    unless defined $device;
+
+  my $host = $device->ip;
+
+  return Status->done("macsuck skipped: $host not yet discovered")
+    unless $device->in_storage;
+
+  return Status->done("macsuck skipped: $host is pseudo-device")
+    if $device->vendor and $device->vendor eq 'netdisco';
+
+  return Status->defer("macsuck deferred: $host is not macsuckable")
+    unless is_macsuckable_now($device);
+
+  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device);
+  return Status->defer("macsuck failed: could not SNMP connect to $host")
+    unless defined $snmp;
+
+  return Status->done("Skipped macsuck for device $host without layer 2 capability")
+    unless $snmp->has_layer(2);
+
+  return Status->done('Macsuck is able to run.');
+});
+
+true;

lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm

@@ -1,62 +1,24 @@
-package App::Netdisco::Core::Macsuck;
+package App::Netdisco::Worker::Plugin::Macsuck::Nodes;
 
-use Dancer qw/:syntax :script/;
+use Dancer ':syntax';
-use Dancer::Plugin::DBIC 'schema';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
 
+use App::Netdisco::Transport::SNMP ();
 use App::Netdisco::Util::Permission 'check_acl_no';
 use App::Netdisco::Util::PortMAC 'get_port_macs';
 use App::Netdisco::Util::Device 'match_devicetype';
 use App::Netdisco::Util::Node 'check_mac';
 use App::Netdisco::Util::SNMP 'snmp_comm_reindex';
+use Dancer::Plugin::DBIC 'schema';
 use Time::HiRes 'gettimeofday';
+use Scope::Guard 'guard';
 
-use base 'Exporter';
+register_worker({ primary => true }, sub {
-our @EXPORT = ();
+  my ($job, $workerconf) = @_;
-our @EXPORT_OK = qw/
-  do_macsuck
-  store_node
-  store_wireless_client_info
-/;
-our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
-=head1 NAME
+  my $device = $job->device;
-
+  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device);
-App::Netdisco::Core::Macsuck
-
-=head1 DESCRIPTION
-
-Helper subroutines to support parts of the Netdisco application.
-
-There are no default exports, however the C<:all> tag will export all
-subroutines.
-
-=head1 EXPORT_OK
-
-=head2 do_macsuck( $device, $snmp )
-
-Given a Device database object, and a working SNMP connection, connect to a
-device and discover the MAC addresses listed against each physical port
-without a neighbor.
-
-If the device has VLANs, C<do_macsuck> will walk each VLAN to get the MAC
-addresses from there.
-
-It will also gather wireless client information if C<store_wireless_clients>
-configuration setting is enabled.
-
-=cut
-
-sub do_macsuck {
-  my ($device, $snmp) = @_;
-
-  unless ($device->in_storage) {
-      debug sprintf
-        ' [%s] macsuck - skipping device not yet discovered',
-        $device->ip;
-      return;
-  }
-  
-  my $ip = $device->ip;
 
   # would be possible just to use now() on updated records, but by using this
   # same value for them all, we can if we want add a job at the end to
@@ -65,7 +27,7 @@ sub do_macsuck {
   my $total_nodes = 0;
 
   # do this before we start messing with the snmp community string
-  store_wireless_client_info($device, $snmp, $now);
+  my $guard = guard { }; # FIXME TODO reset snmp community
 
   # cache the device ports to save hitting the database for many single rows
   my $device_ports = {map {($_->port => $_)}
@@ -74,14 +36,14 @@ sub do_macsuck {
   my $interfaces = $snmp->interfaces;
 
   # get forwarding table data via basic snmp connection
-  my $fwtable = _walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports);
+  my $fwtable = walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports);
 
   # ...then per-vlan if supported
-  my @vlan_list = _get_vlan_list($device, $snmp);
+  my @vlan_list = get_vlan_list($device, $snmp);
   foreach my $vlan (@vlan_list) {
-      snmp_comm_reindex($snmp, $device, $vlan);
+    snmp_comm_reindex($snmp, $device, $vlan);
-      my $pv_fwtable = _walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports, $vlan);
+    my $pv_fwtable = walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports, $vlan);
-      $fwtable = {%$fwtable, %$pv_fwtable};
+    $fwtable = {%$fwtable, %$pv_fwtable};
   }
 
   # now it's time to call store_node for every node discovered
@@ -91,7 +53,7 @@ sub do_macsuck {
   foreach my $vlan (reverse sort keys %$fwtable) {
       foreach my $port (keys %{ $fwtable->{$vlan} }) {
           debug sprintf ' [%s] macsuck - port %s vlan %s : %s nodes',
-            $ip, $port, $vlan, scalar keys %{ $fwtable->{$vlan}->{$port} };
+            $device->ip, $port, $vlan, scalar keys %{ $fwtable->{$vlan}->{$port} };
 
           # make sure this port is UP in netdisco (unless it's a lag master,
           # because we can still see nodes without a functioning aggregate)
@@ -105,30 +67,31 @@ sub do_macsuck {
                 for keys %{ $fwtable->{0} };
 
               ++$total_nodes;
-              store_node($ip, $vlan, $port, $mac, $now);
+              store_node($device->ip, $vlan, $port, $mac, $now);
           }
       }
   }
 
   debug sprintf ' [%s] macsuck - %s updated forwarding table entries',
-    $ip, $total_nodes;
+    $device->ip, $total_nodes;
 
   # a use for $now ... need to archive dissapeared nodes
   my $archived = 0;
 
   if (setting('node_freshness')) {
-      $archived = schema('netdisco')->resultset('Node')->search({
+    $archived = schema('netdisco')->resultset('Node')->search({
-        switch => $ip,
+      switch => $device->ip,
-        time_last => \[ "< ($now - ?::interval)",
+      time_last => \[ "< ($now - ?::interval)",
-          setting('node_freshness') .' minutes' ],
+        setting('node_freshness') .' minutes' ],
-      })->update({ active => \'false' });
+    })->update({ active => \'false' });
   }
 
   debug sprintf ' [%s] macsuck - removed %d fwd table entries to archive',
-    $ip, $archived;
+    $device->ip, $archived;
 
   $device->update({last_macsuck => \$now});
-}
+  return Status->done('Ended macsuck for '. $device->ip);
+});
 
 =head2 store_node( $ip, $vlan, $port, $mac, $now? )
 
@@ -183,7 +146,7 @@ sub store_node {
 }
 
 # return a list of vlan numbers which are OK to macsuck on this device
-sub _get_vlan_list {
+sub get_vlan_list {
   my ($device, $snmp) = @_;
 
   return () unless $snmp->cisco_comm_indexing;
@@ -295,7 +258,7 @@ sub _get_vlan_list {
 
 # walks the forwarding table (BRIDGE-MIB) for the device and returns a
 # table of node entries.
-sub _walk_fwtable {
+sub walk_fwtable {
   my ($device, $snmp, $interfaces, $port_macs, $device_ports, $comm_vlan) = @_;
   my $skiplist = {}; # ports through which we can see another device
   my $cache = {};
@@ -368,8 +331,8 @@ sub _walk_fwtable {
       # do not expose MAC address tables via SNMP. relies on prefetched
       # neighbors otherwise it would kill the DB with device lookups.
       my $neigh_cannot_macsuck = eval { # can fail
-          check_acl_no(($device_port->neighbor || "0 but true"), 'macsuck_unsupported') ||
+        check_acl_no(($device_port->neighbor || "0 but true"), 'macsuck_unsupported') ||
-          match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') };
+        match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') };
 
       if ($device_port->is_uplink) {
           if ($neigh_cannot_macsuck) {
@@ -455,90 +418,4 @@ sub _walk_fwtable {
   return $cache;
 }
 
-=head2 store_wireless_client_info( $device, $snmp, $now? )
+true;
-
-Given a Device database object, and a working SNMP connection, connect to a
-device and discover 802.11 related information for all connected wireless
-clients.
-
-If the device doesn't support the 802.11 MIBs, then this will silently return.
-
-If the device does support the 802.11 MIBs but Netdisco's configuration
-does not permit polling (C<store_wireless_clients> must be true) then a debug
-message is logged and the subroutine returns.
-
-Otherwise, client information is gathered and stored to the database.
-
-Optionally, a third argument can be the literal string passed to the time_last
-field of the database record. If not provided, it defauls to C<now()>.
-
-=cut
-
-sub store_wireless_client_info {
-  my ($device, $snmp, $now) = @_;
-  $now ||= 'now()';
-
-  my $cd11_txrate = $snmp->cd11_txrate;
-  return unless $cd11_txrate and scalar keys %$cd11_txrate;
-
-  if (setting('store_wireless_clients')) {
-      debug sprintf ' [%s] macsuck - gathering wireless client info',
-        $device->ip;
-  }
-  else {
-      debug sprintf ' [%s] macsuck - dot11 info available but skipped due to config',
-        $device->ip;
-      return;
-  }
-
-  my $cd11_rateset = $snmp->cd11_rateset();
-  my $cd11_uptime  = $snmp->cd11_uptime();
-  my $cd11_sigstrength = $snmp->cd11_sigstrength();
-  my $cd11_sigqual = $snmp->cd11_sigqual();
-  my $cd11_mac     = $snmp->cd11_mac();
-  my $cd11_port    = $snmp->cd11_port();
-  my $cd11_rxpkt   = $snmp->cd11_rxpkt();
-  my $cd11_txpkt   = $snmp->cd11_txpkt();
-  my $cd11_rxbyte  = $snmp->cd11_rxbyte();
-  my $cd11_txbyte  = $snmp->cd11_txbyte();
-  my $cd11_ssid    = $snmp->cd11_ssid();
-
-  while (my ($idx, $txrates) = each %$cd11_txrate) {
-      my $rates = $cd11_rateset->{$idx};
-      my $mac   = $cd11_mac->{$idx};
-      next unless defined $mac; # avoid null entries
-            # there can be more rows in txrate than other tables
-
-      my $txrate  = defined $txrates->[$#$txrates]
-        ? int($txrates->[$#$txrates])
-        : undef;
-
-      my $maxrate = defined $rates->[$#$rates]
-        ? int($rates->[$#$rates])
-        : undef;
-
-      my $ssid = $cd11_ssid->{$idx} || 'unknown';
-
-      schema('netdisco')->txn_do(sub {
-        schema('netdisco')->resultset('NodeWireless')
-          ->search({ 'me.mac' => $mac, 'me.ssid' => $ssid })
-          ->update_or_create({
-            txrate  => $txrate,
-            maxrate => $maxrate,
-            uptime  => $cd11_uptime->{$idx},
-            rxpkt   => $cd11_rxpkt->{$idx},
-            txpkt   => $cd11_txpkt->{$idx},
-            rxbyte  => $cd11_rxbyte->{$idx},
-            txbyte  => $cd11_txbyte->{$idx},
-            sigqual => $cd11_sigqual->{$idx},
-            sigstrength => $cd11_sigstrength->{$idx},
-            time_last => \$now,
-          }, {
-            order_by => [qw/mac ssid/],
-            for => 'update',
-          });
-      });
-  }
-}
-
-1;

lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm

@@ -0,0 +1,88 @@
+package App::Netdisco::Worker::Plugin::Macsuck::WirelessNodes;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::Transport::SNMP ();
+use Dancer::Plugin::DBIC 'schema';
+use Time::HiRes 'gettimeofday';
+
+register_worker({ primary => false }, sub {
+  my ($job, $workerconf) = @_;
+
+  my $device = $job->device;
+  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device);
+
+  # would be possible just to use now() on updated records, but by using this
+  # same value for them all, we can if we want add a job at the end to
+  # select and do something with the updated set (see set archive, below)
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  my $cd11_txrate = $snmp->cd11_txrate;
+  return Status->done('Ended macsuck for '. $device->ip)
+    unless $cd11_txrate and scalar keys %$cd11_txrate;
+
+  if (setting('store_wireless_clients')) {
+    debug sprintf ' [%s] macsuck - gathering wireless client info',
+      $device->ip;
+  }
+  else {
+    debug sprintf ' [%s] macsuck - dot11 info available but skipped due to config',
+      $device->ip;
+    return Status->done('Ended macsuck for '. $device->ip);
+  }
+
+  my $cd11_rateset = $snmp->cd11_rateset();
+  my $cd11_uptime  = $snmp->cd11_uptime();
+  my $cd11_sigstrength = $snmp->cd11_sigstrength();
+  my $cd11_sigqual = $snmp->cd11_sigqual();
+  my $cd11_mac     = $snmp->cd11_mac();
+  my $cd11_port    = $snmp->cd11_port();
+  my $cd11_rxpkt   = $snmp->cd11_rxpkt();
+  my $cd11_txpkt   = $snmp->cd11_txpkt();
+  my $cd11_rxbyte  = $snmp->cd11_rxbyte();
+  my $cd11_txbyte  = $snmp->cd11_txbyte();
+  my $cd11_ssid    = $snmp->cd11_ssid();
+
+  while (my ($idx, $txrates) = each %$cd11_txrate) {
+    my $rates = $cd11_rateset->{$idx};
+    my $mac   = $cd11_mac->{$idx};
+    next unless defined $mac; # avoid null entries
+          # there can be more rows in txrate than other tables
+
+    my $txrate  = defined $txrates->[$#$txrates]
+      ? int($txrates->[$#$txrates])
+      : undef;
+
+    my $maxrate = defined $rates->[$#$rates]
+      ? int($rates->[$#$rates])
+      : undef;
+
+    my $ssid = $cd11_ssid->{$idx} || 'unknown';
+
+    schema('netdisco')->txn_do(sub {
+      schema('netdisco')->resultset('NodeWireless')
+        ->search({ 'me.mac' => $mac, 'me.ssid' => $ssid })
+        ->update_or_create({
+          txrate  => $txrate,
+          maxrate => $maxrate,
+          uptime  => $cd11_uptime->{$idx},
+          rxpkt   => $cd11_rxpkt->{$idx},
+          txpkt   => $cd11_txpkt->{$idx},
+          rxbyte  => $cd11_rxbyte->{$idx},
+          txbyte  => $cd11_txbyte->{$idx},
+          sigqual => $cd11_sigqual->{$idx},
+          sigstrength => $cd11_sigstrength->{$idx},
+          time_last => \$now,
+        }, {
+          order_by => [qw/mac ssid/],
+          for => 'update',
+        });
+    });
+  }
+
+  return Status->done('Ended macsuck for '. $device->ip);
+});
+
+true;

lib/App/Netdisco/Worker/Plugin/Macwalk.pm

@@ -0,0 +1,31 @@
+package App::Netdisco::Worker::Plugin::Macwalk;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use Dancer::Plugin::DBIC 'schema';
+
+register_worker({ primary => true }, sub {
+  my ($job, $workerconf) = @_;
+
+  my %queued = map {$_ => 1} jq_queued('macsuck');
+  my @devices = schema('netdisco')->resultset('Device')->search({
+    -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
+  })->has_layer('2')->get_column('ip')->all;
+  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+
+  jq_insert([
+    map {{
+      device => $_,
+      action => 'macsuck',
+      username => $job->username,
+      userip => $job->userip,
+    }} (@filtered_devices)
+  ]);
+
+  return Status->done('Queued macsuck job for all devices');
+});
+
+true;

lib/App/Netdisco/Worker/Plugin/Nbtstat.pm

@@ -10,7 +10,7 @@ register_worker({ primary => true }, sub {
   my ($job, $workerconf) = @_;
 
   return Status->error('nbtstat failed: unable to interpret device param')
-    if !defined $job->device;
+    unless defined $job->device;
 
   return Status->defer(sprintf 'nbtstat deferred: %s is not macsuckable', $job->device->ip)
     unless is_macsuckable($job->device);