Support for expire_devices, expire_nodes, and expire_nodes_archive

Netdisco/Changes

@@ -7,6 +7,7 @@
   * [#3] [#47] Device Neighbor Map can have max depth and VLAN filter
   * [#31] get_community now supported
   * [#19] Ask for Reason when changing Port up/down Status, or VLAN
+  * [#30] Support for expire_devices, expire_nodes, and expire_nodes_archive
 
   [ENHANCEMENTS]
 

Netdisco/bin/netdisco-do

@@ -75,6 +75,7 @@ if (!length $action) {
   with 'App::Netdisco::Daemon::Worker::Poller::Device';
   with 'App::Netdisco::Daemon::Worker::Poller::Arpnip';
   with 'App::Netdisco::Daemon::Worker::Poller::Macsuck';
+  with 'App::Netdisco::Daemon::Worker::Poller::Expiry';
   with 'App::Netdisco::Daemon::Worker::Interactive::DeviceActions';
   with 'App::Netdisco::Daemon::Worker::Interactive::PortActions';
 }

Netdisco/lib/App/Netdisco/Daemon/Queue.pm

@@ -22,7 +22,7 @@ sub capacity_for {
   debug "checking local capacity for action $action";
 
   my $action_map = {
-    Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck/],
+    Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck expiry/],
     Interactive => [qw/location contact portcontrol portname vlan power/],
   };
 

Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm

@@ -13,7 +13,7 @@ my $fqdn = hostfqdn || 'localhost';
 
 my $role_map = {
   (map {$_ => 'Poller'}
-      qw/discoverall discover arpwalk arpnip macwalk macsuck/),
+      qw/discoverall discover arpwalk arpnip macwalk macsuck expiry/),
   (map {$_ => 'Interactive'}
       qw/location contact portcontrol portname vlan power/)
 };

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm

@@ -9,7 +9,8 @@ with 'App::Netdisco::Daemon::Worker::Common';
 # add dispatch methods for poller tasks
 with 'App::Netdisco::Daemon::Worker::Poller::Device',
      'App::Netdisco::Daemon::Worker::Poller::Arpnip',
-     'App::Netdisco::Daemon::Worker::Poller::Macsuck';
+     'App::Netdisco::Daemon::Worker::Poller::Macsuck',
+     'App::Netdisco::Daemon::Worker::Poller::Expiry';
 
 sub worker_type { 'pol' }
 sub worker_name { 'Poller' }

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Expiry.pm

@@ -0,0 +1,46 @@
+package App::Netdisco::Daemon::Worker::Poller::Expiry;
+
+use Dancer qw/:moose :syntax :script/;
+use Dancer::Plugin::DBIC 'schema';
+
+use App::Netdisco::Daemon::Util ':all';
+
+use Role::Tiny;
+use namespace::clean;
+
+# expire devices and nodes according to config
+sub expiry {
+  my ($self, $job) = @_;
+
+  if (setting('expire_devices') and setting('expire_devices') > 0) {
+      schema('netdisco')->txn_do(sub {
+        schema('netdisco')->resultset('Device')->search({
+          last_discover => \[q/< (now() - ?::interval)/,
+              (setting('expire_devices') * 86400)],
+        })->delete();
+      });
+  }
+
+  if (setting('expire_nodes') and setting('expire_nodes') > 0) {
+      schema('netdisco')->txn_do(sub {
+        schema('netdisco')->resultset('Node')->search({
+          time_last => \[q/< (now() - ?::interval)/,
+              (setting('expire_nodes') * 86400)],
+        })->delete();
+      });
+  }
+
+  if (setting('expire_nodes_archive') and setting('expire_nodes_archive') > 0) {
+      schema('netdisco')->txn_do(sub {
+        schema('netdisco')->resultset('Node')->search({
+          -not_bool => 'active',
+          time_last => \[q/< (now() - ?::interval)/,
+              (setting('expire_nodes_archive') * 86400)],
+        })->delete();
+      });
+  }
+
+  return job_done("Checked expiry for all Devices and Nodes");
+}
+
+1;

Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm

@@ -14,6 +14,7 @@ my $jobactions = {
       discoverall
       arpwalk
       macwalk
+      expiry
   /
 #    saveconfigs
 #    nbtwalk

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -555,6 +555,28 @@ Value: Number. Default: 0.
 Sets the minimum amount of time in seconds which must elapse between any two
 arpnip jobs for a device.
 
+=head3 C<expire_devices>
+
+Value: Number of Days.
+
+Devices that have not been refreshed in this number of days will be removed.
+All nodes connected to this device will be removed as well.
+
+=head3 C<expire_nodes>
+
+Value: Number of Days.
+
+Nodes that have not been refreshed in this number of days will be removed from
+the database. Archived and non-archived nodes are removed. This includes
+SwitchPort/MAC and MAC/IP mappings.
+
+=head3 C<expire_nodes_archive>
+
+Value: Number of Days.
+
+Archived data for switch-port/MAC and MAC/IP mappings older than this number
+of days will be removed.
+
 =head3 C<dns>
 
 Value: Settings Tree. Default: C<max_outstanding: 250>
@@ -741,9 +763,11 @@ hour fields (which accept same types as C<cron> notation). For example:
        min: 15
        hour: '*/2'
        wday: 'mon-fri'
+   expiry:
+     when: '20 23 * * *'
 
-Note that the fields default to "all" (i.e. "C<*>") when not specified. See
-L<Algorithm::Cron> for further details.
+Note that the "C<when>" fields default to "all" (i.e. "C<*>") when not
+specified. See L<Algorithm::Cron> for further details.
 
 =head2 Dancer Internal
 
@@ -837,18 +861,6 @@ C<col_xxx_show>
 
 =item *
 
-C<expire_devices>
-
-=item *
-
-C<expire_nodes>
-
-=item *
-
-C<expire_nodes_archive>
-
-=item *
-
 C<macsuck_timeout>
 
 =item *

Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod

@@ -41,10 +41,31 @@ but they are backwards compatible.
 
 SNMP community strings provided in the C<community_rw> configuration setting
 will I<no longer> be used for I<read> actions on a device (despite having
-"C<rw>" in the setting name). If you have the same community string for read
-and write access, then you must set both C<community> and C<community_rw> in
-your C<deployment.yml> file. In any case, we recommend using the new
-C<snmp_auth> configuration format which supercedes both these settings.
+"C<rw>" in the setting name).
+
+If you have the same community string for read and write access, then you must
+set both C<community> and C<community_rw> in your C<deployment.yml> file. In
+any case, we recommend using the new C<snmp_auth> configuration format which
+supercedes both these settings.
+
+=head2 Health Advice
+
+This release includes support for Device and Node expiry from your database.
+This is an important part of housekeeping for your installation, and our
+recommendation is to enable this feature such that suitably old Devices and
+Nodes are expired nightly.
+
+Add the following to your "C<housekeeping>" configuration in
+C<deployment.yml>, to have a nightly check at 11:20pm:
+
+  housekeeping:
+    expiry:
+      when: '20 23 * * *'
+
+You should also configure one or more of C<expire_devices>, C<expire_nodes>,
+and C<expire_nodes_archive> to a number of days. See the
+L<Configuration|App::Netdisco::Manual::Configuration> documentation for
+further details.
 
 =head1 2.020000
 
@@ -95,7 +116,7 @@ version there will be a way to display them in the web interface.
 
 =head1 2.015000
 
-=head2 Heath Advice
+=head2 Health Advice
 
 Some of the "dangerous action" confirmation dialogs offer to take a log
 message (e.g. Port Control, Device Delete). Currently the log messages are
@@ -162,7 +183,7 @@ Deployment and Configuration documentation for further details.
 
 =head1 2.008000
 
-=head2 Heath Advice
+=head2 Health Advice
 
 This release contains the first version of our new poller, which handles
 device and node discovery. Please make sure to backup any existing Netdisco

Netdisco/share/config.yml

@@ -99,6 +99,9 @@ macsuck_min_age: 0
 arpnip_no: []
 arpnip_only: []
 arpnip_min_age: 0
+expire_devices: 0
+expire_nodes: 0
+expire_nodes_archive: 0
 store_wireless_clients: true
 store_modules: true
 ignore_interfaces:

Netdisco/share/environments/deployment.yml

@@ -43,4 +43,9 @@ database:
 #   arpwalk:
 #     when:
 #       min: 50
+#   expiry:
+#     when: '20 23 * * *'
 
+# expire_devices: 60
+# expire_nodes: 90
+# expire_nodes_archive: 60