From c6ebb7cf0717cd4f1861ed997846dcfaeebd42b7 Mon Sep 17 00:00:00 2001
From: Oliver Gorwits <oliver@cpan.org>
Date: Sat, 9 Sep 2017 16:44:32 +0100
Subject: [PATCH] move arpnip and arpwalk to worker plugins

---
 .../Netdisco/Backend/Worker/Poller/Arpnip.pm  |  18 ---
 .../Netdisco/Backend/Worker/Poller/Common.pm  |  99 ----------------
 lib/App/Netdisco/Worker/Plugin/Arpnip.pm      |  31 +++++
 .../Plugin/Arpnip/Nodes.pm}                   | 110 +++---------------
 .../Netdisco/Worker/Plugin/Arpnip/Subnets.pm  |  75 ++++++++++++
 lib/App/Netdisco/Worker/Plugin/Arpwalk.pm     |  31 +++++
 lib/App/Netdisco/Worker/Plugin/Macsuck.pm     |   2 +-
 7 files changed, 157 insertions(+), 209 deletions(-)
 delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm
 delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Common.pm
 create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpnip.pm
 rename lib/App/Netdisco/{Core/Arpnip.pm => Worker/Plugin/Arpnip/Nodes.pm} (52%)
 create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm
 create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpwalk.pm

diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm b/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm
deleted file mode 100644
index fb2ce593..00000000
--- a/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm
+++ /dev/null
@@ -1,18 +0,0 @@
-package App::Netdisco::Backend::Worker::Poller::Arpnip;
-
-use App::Netdisco::Core::Arpnip 'do_arpnip';
-use App::Netdisco::Util::Device 'is_arpnipable_now';
-
-use Role::Tiny;
-use namespace::clean;
-
-with 'App::Netdisco::Backend::Worker::Poller::Common';
-
-sub arpnip_action { \&do_arpnip }
-sub arpnip_filter { \&is_arpnipable_now }
-sub arpnip_layer { 3 }
-
-sub arpwalk { (shift)->_walk_body('arpnip', @_) }
-sub arpnip  { (shift)->_single_body('arpnip', @_) }
-
-1;
diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Common.pm b/lib/App/Netdisco/Backend/Worker/Poller/Common.pm
deleted file mode 100644
index 1489b4a4..00000000
--- a/lib/App/Netdisco/Backend/Worker/Poller/Common.pm
+++ /dev/null
@@ -1,99 +0,0 @@
-package App::Netdisco::Backend::Worker::Poller::Common;
-
-use Dancer qw/:moose :syntax :script/;
-
-use App::Netdisco::Transport::SNMP;
-use App::Netdisco::Util::Device 'get_device';
-use App::Netdisco::Backend::Util ':all';
-use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
-
-use Dancer::Plugin::DBIC 'schema';
-use NetAddr::IP::Lite ':lower';
-
-use Role::Tiny;
-use namespace::clean;
-
-# queue a job for all devices known to Netdisco
-sub _walk_body {
-  my ($self, $job_type, $job) = @_;
-
-  my $layer_method = $job_type .'_layer';
-  my $job_layer = $self->$layer_method;
-
-  my %queued = map {$_ => 1} jq_queued($job_type);
-  my @devices = schema('netdisco')->resultset('Device')->search({
-    -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
-  })->has_layer($job_layer)->get_column('ip')->all;
-  my @filtered_devices = grep {!exists $queued{$_}} @devices;
-
-  jq_insert([
-      map {{
-          device => $_,
-          action => $job_type,
-          username => $job->username,
-          userip => $job->userip,
-      }} (@filtered_devices)
-  ]);
-
-  return job_done("Queued $job_type job for all devices");
-}
-
-sub _single_body {
-  my ($self, $job_type, $job) = @_;
-
-  my $action_method = $job_type .'_action';
-  my $job_action = $self->$action_method;
-
-  my $layer_method = $job_type .'_layer';
-  my $job_layer = $self->$layer_method;
-
-  my $device = get_device($job->device)
-    or job_error("$job_type failed: unable to interpret device parameter");
-  my $host = $device->ip;
-
-  if ($device->in_storage
-      and $device->vendor and $device->vendor eq 'netdisco') {
-      return job_done("$job_type skipped: $host is pseudo-device");
-  }
-
-  my $filter_method = $job_type .'_filter';
-  my $job_filter = $self->$filter_method;
-
-  unless ($job_filter->($device->ip)) {
-      return job_defer("$job_type deferred: $host is not ${job_type}able");
-  }
-
-  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device);
-  if (!defined $snmp) {
-      return job_defer("$job_type failed: could not SNMP connect to $host");
-  }
-
-  unless ($snmp->has_layer( $job_layer )) {
-      return job_done("Skipped $job_type for device $host without OSI layer $job_layer capability");
-  }
-
-  $job_action->($device, $snmp);
-
-  return job_done("Ended $job_type for $host");
-}
-
-sub _single_node_body {
-  my ($self, $job_type, $node, $now) = @_;
-
-  my $action_method = $job_type .'_action';
-  my $job_action = $self->$action_method;
-
-  my $filter_method = $job_type .'_filter';
-  my $job_filter = $self->$filter_method;
-
-  unless ($job_filter->($node)) {
-      return job_defer("$job_type deferred: $node is not ${job_type}able");
-  }
-
-  $job_action->($node, $now);
-
-  # would be ignored if wrapped in a loop
-  return job_done("Ended $job_type for $node");
-}
-
-1;
diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip.pm
new file mode 100644
index 00000000..74c70bfc
--- /dev/null
+++ b/lib/App/Netdisco/Worker/Plugin/Arpnip.pm
@@ -0,0 +1,31 @@
+package App::Netdisco::Worker::Plugin::Arpnip;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::Util::Device 'is_arpnipable_now';
+use App::Netdisco::Transport::SNMP ();
+
+register_worker({ primary => true }, sub {
+  my ($job, $workerconf) = @_;
+  my $device = $job->device;
+
+  return Status->error('arpnip failed: unable to interpret device param')
+    unless defined $device;
+
+  my $host = $device->ip;
+
+  return Status->done("arpnip skipped: $host not yet discovered")
+    unless $device->in_storage;
+
+  return Status->defer("arpnip skipped: $host is pseudo-device")
+    if $device->vendor and $device->vendor eq 'netdisco';
+
+  return Status->defer("arpnip deferred: $host is not arpnipable")
+    unless is_arpnipable_now($device);
+
+  return Status->done('Arpnip is able to run.');
+});
+
+true;
diff --git a/lib/App/Netdisco/Core/Arpnip.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
similarity index 52%
rename from lib/App/Netdisco/Core/Arpnip.pm
rename to lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
index 335fff87..13c98ded 100644
--- a/lib/App/Netdisco/Core/Arpnip.pm
+++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm
@@ -1,58 +1,29 @@
-package App::Netdisco::Core::Arpnip;
+package App::Netdisco::Worker::Plugin::Arpnip::Nodes;
 
-use Dancer qw/:syntax :script/;
-use Dancer::Plugin::DBIC 'schema';
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
 
 use App::Netdisco::Util::Node 'check_mac';
-use App::Netdisco::Util::Permission 'check_acl_no';
 use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
-use NetAddr::IP::Lite ':lower';
+use Dancer::Plugin::DBIC 'schema';
 use Time::HiRes 'gettimeofday';
 use NetAddr::MAC ();
 
-use base 'Exporter';
-our @EXPORT = ();
-our @EXPORT_OK = qw/ do_arpnip store_arp /;
-our %EXPORT_TAGS = (all => \@EXPORT_OK);
+register_worker({ primary => true, driver => 'snmp' }, sub {
+  my ($job, $workerconf) = @_;
 
-=head1 NAME
+  my $device = $job->device;
+  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
+    or return Status->defer("arpnip failed: could not SNMP connect to $host");
 
-App::Netdisco::Core::Arpnip
-
-=head1 DESCRIPTION
-
-Helper subroutines to support parts of the Netdisco application.
-
-There are no default exports, however the C<:all> tag will export all
-subroutines.
-
-=head1 EXPORT_OK
-
-=head2 do_arpnip( $device, $snmp )
-
-Given a Device database object, and a working SNMP connection, connect to a
-device and discover its ARP cache for IPv4 and Neighbor cache for IPv6.
-
-Will also discover subnets in use on the device and update the Subnets table.
-
-=cut
-
-sub do_arpnip {
-  my ($device, $snmp) = @_;
-
-  unless ($device->in_storage) {
-      debug sprintf ' [%s] arpnip - skipping device not yet discovered', $device->ip;
-      return;
-  }
+  return Status->defer("Skipped arpnip for device $host without layer 3 capability")
+    unless $snmp->has_layer(3);
 
   # get v4 arp table
-  my $v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
+  my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr);
   # get v6 neighbor cache
-  my $v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
-
-  # get directly connected networks
-  my @subnets = _gather_subnets($device, $snmp);
-  # TODO: IPv6 subnets
+  my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr);
 
   # would be possible just to use now() on updated records, but by using this
   # same value for them all, we _can_ if we want add a job at the end to
@@ -68,15 +39,12 @@ sub do_arpnip {
   debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries',
     $device->ip, scalar @$v6;
 
-  _store_subnet($_, $now) for @subnets;
-  debug sprintf ' [%s] arpnip - processed %s Subnet entries',
-    $device->ip, scalar @subnets;
-
   $device->update({last_arpnip => \$now});
-}
+  return Status->done('Ended arpnip for '. $device->ip);
+});
 
 # get an arp table (v4 or v6)
-sub _get_arps {
+sub get_arps {
   my ($device, $paddr, $netaddr) = @_;
   my @arps = ();
 
@@ -92,7 +60,7 @@ sub _get_arps {
   }
 
   debug sprintf ' resolving %d ARP entries with max %d outstanding requests',
-      scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
+    scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'};
   my $resolved_ips = hostnames_resolve_async(\@arps);
 
   return $resolved_ips;
@@ -148,44 +116,4 @@ sub store_arp {
   });
 }
 
-# gathers device subnets
-sub _gather_subnets {
-  my ($device, $snmp) = @_;
-  my @subnets = ();
-
-  my $ip_netmask = $snmp->ip_netmask;
-  foreach my $entry (keys %$ip_netmask) {
-      my $ip = NetAddr::IP::Lite->new($entry);
-      my $addr = $ip->addr;
-
-      next if $addr eq '0.0.0.0';
-      next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__');
-      next if setting('ignore_private_nets') and $ip->is_rfc1918;
-
-      my $netmask = $ip_netmask->{$addr};
-      next if $netmask eq '255.255.255.255' or $netmask eq '0.0.0.0';
-
-      my $cidr = NetAddr::IP::Lite->new($addr, $netmask)->network->cidr;
-
-      debug sprintf ' [%s] arpnip - found subnet %s', $device->ip, $cidr;
-      push @subnets, $cidr;
-  }
-
-  return @subnets;
-}
-
-# update subnets with new networks
-sub _store_subnet {
-  my ($subnet, $now) = @_;
-
-  schema('netdisco')->txn_do(sub {
-    schema('netdisco')->resultset('Subnet')->update_or_create(
-    {
-      net => $subnet,
-      last_discover => \$now,
-    },
-    { for => 'update' });
-  });
-}
-
-1;
+true;
diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm
new file mode 100644
index 00000000..c523ecac
--- /dev/null
+++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm
@@ -0,0 +1,75 @@
+package App::Netdisco::Worker::Plugin::Arpnip::Subnets;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::Util::Permission 'check_acl_no';
+use Dancer::Plugin::DBIC 'schema';
+use NetAddr::IP::Lite ':lower';
+use Time::HiRes 'gettimeofday';
+
+register_worker({ primary => false, driver => 'snmp' }, sub {
+  my ($job, $workerconf) = @_;
+
+  my $device = $job->device;
+  my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
+    or return Status->defer("arpnip failed: could not SNMP connect to $host");
+
+  return Status->defer("Skipped arpnip for device $host without layer 3 capability")
+    unless $snmp->has_layer(3);
+
+  # get directly connected networks
+  my @subnets = gather_subnets($device, $snmp);
+  # TODO: IPv6 subnets
+
+  my $now = 'to_timestamp('. (join '.', gettimeofday) .')';
+
+  store_subnet($_, $now) for @subnets;
+  debug sprintf ' [%s] arpnip - processed %s Subnet entries',
+    $device->ip, scalar @subnets;
+
+  return Status->done('Ended arpnip for '. $device->ip);
+});
+
+# gathers device subnets
+sub gather_subnets {
+  my ($device, $snmp) = @_;
+  my @subnets = ();
+
+  my $ip_netmask = $snmp->ip_netmask;
+  foreach my $entry (keys %$ip_netmask) {
+      my $ip = NetAddr::IP::Lite->new($entry);
+      my $addr = $ip->addr;
+
+      next if $addr eq '0.0.0.0';
+      next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__');
+      next if setting('ignore_private_nets') and $ip->is_rfc1918;
+
+      my $netmask = $ip_netmask->{$addr};
+      next if $netmask eq '255.255.255.255' or $netmask eq '0.0.0.0';
+
+      my $cidr = NetAddr::IP::Lite->new($addr, $netmask)->network->cidr;
+
+      debug sprintf ' [%s] arpnip - found subnet %s', $device->ip, $cidr;
+      push @subnets, $cidr;
+  }
+
+  return @subnets;
+}
+
+# update subnets with new networks
+sub store_subnet {
+  my ($subnet, $now) = @_;
+
+  schema('netdisco')->txn_do(sub {
+    schema('netdisco')->resultset('Subnet')->update_or_create(
+    {
+      net => $subnet,
+      last_discover => \$now,
+    },
+    { for => 'update' });
+  });
+}
+
+true;
diff --git a/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm
new file mode 100644
index 00000000..44ee47bc
--- /dev/null
+++ b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm
@@ -0,0 +1,31 @@
+package App::Netdisco::Worker::Plugin::Arpwalk;
+
+use Dancer ':syntax';
+use App::Netdisco::Worker::Plugin;
+use aliased 'App::Netdisco::Worker::Status';
+
+use App::Netdisco::JobQueue qw/jq_queued jq_insert/;
+use Dancer::Plugin::DBIC 'schema';
+
+register_worker({ primary => true }, sub {
+  my ($job, $workerconf) = @_;
+
+  my %queued = map {$_ => 1} jq_queued('arpnip');
+  my @devices = schema('netdisco')->resultset('Device')->search({
+    -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }],
+  })->has_layer('3')->get_column('ip')->all;
+  my @filtered_devices = grep {!exists $queued{$_}} @devices;
+
+  jq_insert([
+    map {{
+      device => $_,
+      action => 'arpnip',
+      username => $job->username,
+      userip => $job->userip,
+    }} (@filtered_devices)
+  ]);
+
+  return Status->done('Queued arpnip job for all devices');
+});
+
+true;
diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck.pm
index c698cc8d..7ac98cf9 100644
--- a/lib/App/Netdisco/Worker/Plugin/Macsuck.pm
+++ b/lib/App/Netdisco/Worker/Plugin/Macsuck.pm
@@ -19,7 +19,7 @@ register_worker({ primary => true }, sub {
   return Status->done("macsuck skipped: $host not yet discovered")
     unless $device->in_storage;
 
-  return Status->done("macsuck skipped: $host is pseudo-device")
+  return Status->defer("macsuck skipped: $host is pseudo-device")
     if $device->vendor and $device->vendor eq 'netdisco';
 
   return Status->defer("macsuck deferred: $host is not macsuckable")