From 183bf81cc62f4840d39e7ea36f8a277c1de2eb40 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sun, 29 Dec 2013 19:29:59 +0000 Subject: [PATCH] Support for expire_devices, expire_nodes, and expire_nodes_archive --- Netdisco/Changes | 1 + Netdisco/bin/netdisco-do | 1 + Netdisco/lib/App/Netdisco/Daemon/Queue.pm | 2 +- .../lib/App/Netdisco/Daemon/Worker/Manager.pm | 2 +- .../lib/App/Netdisco/Daemon/Worker/Poller.pm | 3 +- .../Netdisco/Daemon/Worker/Poller/Expiry.pm | 46 +++++++++++++++++++ .../App/Netdisco/Daemon/Worker/Scheduler.pm | 1 + .../lib/App/Netdisco/Manual/Configuration.pod | 40 ++++++++++------ .../lib/App/Netdisco/Manual/ReleaseNotes.pod | 33 ++++++++++--- Netdisco/share/config.yml | 3 ++ Netdisco/share/environments/deployment.yml | 5 ++ 11 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Expiry.pm diff --git a/Netdisco/Changes b/Netdisco/Changes index e1eb8094..8446a7e8 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -7,6 +7,7 @@ * [#3] [#47] Device Neighbor Map can have max depth and VLAN filter * [#31] get_community now supported * [#19] Ask for Reason when changing Port up/down Status, or VLAN + * [#30] Support for expire_devices, expire_nodes, and expire_nodes_archive [ENHANCEMENTS] diff --git a/Netdisco/bin/netdisco-do b/Netdisco/bin/netdisco-do index 6e2fd1f2..1d691145 100755 --- a/Netdisco/bin/netdisco-do +++ b/Netdisco/bin/netdisco-do @@ -75,6 +75,7 @@ if (!length $action) { with 'App::Netdisco::Daemon::Worker::Poller::Device'; with 'App::Netdisco::Daemon::Worker::Poller::Arpnip'; with 'App::Netdisco::Daemon::Worker::Poller::Macsuck'; + with 'App::Netdisco::Daemon::Worker::Poller::Expiry'; with 'App::Netdisco::Daemon::Worker::Interactive::DeviceActions'; with 'App::Netdisco::Daemon::Worker::Interactive::PortActions'; } diff --git a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm index f89e14fb..811a8f2a 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm @@ -22,7 +22,7 @@ sub capacity_for { debug "checking local capacity for action $action"; my $action_map = { - Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck/], + Poller => [qw/discoverall discover arpwalk arpnip macwalk macsuck expiry/], Interactive => [qw/location contact portcontrol portname vlan power/], }; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index 047196f2..e5dc5b63 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -13,7 +13,7 @@ my $fqdn = hostfqdn || 'localhost'; my $role_map = { (map {$_ => 'Poller'} - qw/discoverall discover arpwalk arpnip macwalk macsuck/), + qw/discoverall discover arpwalk arpnip macwalk macsuck expiry/), (map {$_ => 'Interactive'} qw/location contact portcontrol portname vlan power/) }; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm index 8df43f51..499de226 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm @@ -9,7 +9,8 @@ with 'App::Netdisco::Daemon::Worker::Common'; # add dispatch methods for poller tasks with 'App::Netdisco::Daemon::Worker::Poller::Device', 'App::Netdisco::Daemon::Worker::Poller::Arpnip', - 'App::Netdisco::Daemon::Worker::Poller::Macsuck'; + 'App::Netdisco::Daemon::Worker::Poller::Macsuck', + 'App::Netdisco::Daemon::Worker::Poller::Expiry'; sub worker_type { 'pol' } sub worker_name { 'Poller' } diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Expiry.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Expiry.pm new file mode 100644 index 00000000..86156716 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Expiry.pm @@ -0,0 +1,46 @@ +package App::Netdisco::Daemon::Worker::Poller::Expiry; + +use Dancer qw/:moose :syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Daemon::Util ':all'; + +use Role::Tiny; +use namespace::clean; + +# expire devices and nodes according to config +sub expiry { + my ($self, $job) = @_; + + if (setting('expire_devices') and setting('expire_devices') > 0) { + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('Device')->search({ + last_discover => \[q/< (now() - ?::interval)/, + (setting('expire_devices') * 86400)], + })->delete(); + }); + } + + if (setting('expire_nodes') and setting('expire_nodes') > 0) { + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('Node')->search({ + time_last => \[q/< (now() - ?::interval)/, + (setting('expire_nodes') * 86400)], + })->delete(); + }); + } + + if (setting('expire_nodes_archive') and setting('expire_nodes_archive') > 0) { + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('Node')->search({ + -not_bool => 'active', + time_last => \[q/< (now() - ?::interval)/, + (setting('expire_nodes_archive') * 86400)], + })->delete(); + }); + } + + return job_done("Checked expiry for all Devices and Nodes"); +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm index cce5f282..63255ee4 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm @@ -14,6 +14,7 @@ my $jobactions = { discoverall arpwalk macwalk + expiry / # saveconfigs # nbtwalk diff --git a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod index b4119802..795c185d 100644 --- a/Netdisco/lib/App/Netdisco/Manual/Configuration.pod +++ b/Netdisco/lib/App/Netdisco/Manual/Configuration.pod @@ -555,6 +555,28 @@ Value: Number. Default: 0. Sets the minimum amount of time in seconds which must elapse between any two arpnip jobs for a device. +=head3 C + +Value: Number of Days. + +Devices that have not been refreshed in this number of days will be removed. +All nodes connected to this device will be removed as well. + +=head3 C + +Value: Number of Days. + +Nodes that have not been refreshed in this number of days will be removed from +the database. Archived and non-archived nodes are removed. This includes +SwitchPort/MAC and MAC/IP mappings. + +=head3 C + +Value: Number of Days. + +Archived data for switch-port/MAC and MAC/IP mappings older than this number +of days will be removed. + =head3 C Value: Settings Tree. Default: C @@ -741,9 +763,11 @@ hour fields (which accept same types as C notation). For example: min: 15 hour: '*/2' wday: 'mon-fri' + expiry: + when: '20 23 * * *' -Note that the fields default to "all" (i.e. "C<*>") when not specified. See -L for further details. +Note that the "C" fields default to "all" (i.e. "C<*>") when not +specified. See L for further details. =head2 Dancer Internal @@ -837,18 +861,6 @@ C =item * -C - -=item * - -C - -=item * - -C - -=item * - C =item * diff --git a/Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod b/Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod index db2439c2..c464f4a7 100644 --- a/Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod +++ b/Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod @@ -41,10 +41,31 @@ but they are backwards compatible. SNMP community strings provided in the C configuration setting will I be used for I actions on a device (despite having -"C" in the setting name). If you have the same community string for read -and write access, then you must set both C and C in -your C file. In any case, we recommend using the new -C configuration format which supercedes both these settings. +"C" in the setting name). + +If you have the same community string for read and write access, then you must +set both C and C in your C file. In +any case, we recommend using the new C configuration format which +supercedes both these settings. + +=head2 Health Advice + +This release includes support for Device and Node expiry from your database. +This is an important part of housekeeping for your installation, and our +recommendation is to enable this feature such that suitably old Devices and +Nodes are expired nightly. + +Add the following to your "C" configuration in +C, to have a nightly check at 11:20pm: + + housekeeping: + expiry: + when: '20 23 * * *' + +You should also configure one or more of C, C, +and C to a number of days. See the +L documentation for +further details. =head1 2.020000 @@ -95,7 +116,7 @@ version there will be a way to display them in the web interface. =head1 2.015000 -=head2 Heath Advice +=head2 Health Advice Some of the "dangerous action" confirmation dialogs offer to take a log message (e.g. Port Control, Device Delete). Currently the log messages are @@ -162,7 +183,7 @@ Deployment and Configuration documentation for further details. =head1 2.008000 -=head2 Heath Advice +=head2 Health Advice This release contains the first version of our new poller, which handles device and node discovery. Please make sure to backup any existing Netdisco diff --git a/Netdisco/share/config.yml b/Netdisco/share/config.yml index b93aabc7..5589bd75 100644 --- a/Netdisco/share/config.yml +++ b/Netdisco/share/config.yml @@ -99,6 +99,9 @@ macsuck_min_age: 0 arpnip_no: [] arpnip_only: [] arpnip_min_age: 0 +expire_devices: 0 +expire_nodes: 0 +expire_nodes_archive: 0 store_wireless_clients: true store_modules: true ignore_interfaces: diff --git a/Netdisco/share/environments/deployment.yml b/Netdisco/share/environments/deployment.yml index 0409198c..d5b5ed3f 100644 --- a/Netdisco/share/environments/deployment.yml +++ b/Netdisco/share/environments/deployment.yml @@ -43,4 +43,9 @@ database: # arpwalk: # when: # min: 50 +# expiry: +# when: '20 23 * * *' +# expire_devices: 60 +# expire_nodes: 90 +# expire_nodes_archive: 60