diff --git a/Netdisco/Changes b/Netdisco/Changes index 149085d3..b13b6360 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -1,5 +1,13 @@ 2.007000_002 - + * NOTE this version requires SNMP::Info 3.x + + [NEW FEATURES] + + * Finally we have a discover/refresh daemon job :) + * Also... a Scheduler which removes need for crontab installation + * The netdisco-do script can run a one-off discover for a device + [BUG FIXES] * Rename plugins developer doc to .pod diff --git a/Netdisco/MANIFEST b/Netdisco/MANIFEST index 77c75e66..75ea3a6d 100644 --- a/Netdisco/MANIFEST +++ b/Netdisco/MANIFEST @@ -3,6 +3,7 @@ bin/netdisco-daemon bin/netdisco-daemon-fg bin/netdisco-db-deploy bin/netdisco-deploy +bin/netdisco-do bin/netdisco-web bin/netdisco-web-fg Changes @@ -25,6 +26,9 @@ lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm lib/App/Netdisco/Daemon/Worker/Interactive/Util.pm lib/App/Netdisco/Daemon/Worker/Manager.pm +lib/App/Netdisco/Daemon/Worker/Poller.pm +lib/App/Netdisco/Daemon/Worker/Poller/Discover.pm +lib/App/Netdisco/Daemon/Worker/Scheduler.pm lib/App/Netdisco/DB.pm lib/App/Netdisco/DB/Result/Admin.pm lib/App/Netdisco/DB/Result/Device.pm @@ -71,7 +75,9 @@ lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-12-13-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-13-14-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-14-15-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-15-16-PostgreSQL.sql +lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-16-17-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-16-PostgreSQL.sql +lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-17-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-3-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-PostgreSQL.sql lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-3-4-PostgreSQL.sql @@ -85,10 +91,12 @@ lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-9-10-PostgreSQL.sql lib/App/Netdisco/Manual/Deployment.pod lib/App/Netdisco/Manual/Developing.pod lib/App/Netdisco/Manual/ReleaseNotes.pod -lib/App/Netdisco/Manual/WritingPlugins.pm -lib/App/Netdisco/Util/Connect.pm -lib/App/Netdisco/Util/DeviceProperties.pm -lib/App/Netdisco/Util/Permissions.pm +lib/App/Netdisco/Manual/WritingPlugins.pod +lib/App/Netdisco/Util/Device.pm +lib/App/Netdisco/Util/DiscoverAndStore.pm +lib/App/Netdisco/Util/DNS.pm +lib/App/Netdisco/Util/Port.pm +lib/App/Netdisco/Util/SNMP.pm lib/App/Netdisco/Util/Web.pm lib/App/Netdisco/Web.pm lib/App/Netdisco/Web/AuthN.pm diff --git a/Netdisco/META.yml b/Netdisco/META.yml index 071fa3d9..78d68a42 100644 --- a/Netdisco/META.yml +++ b/Netdisco/META.yml @@ -19,6 +19,7 @@ no_index: - inc - share requires: + Algorithm::Cron: 0 App::cpanminus: 0 App::local::lib::helper: 0 DBD::Pg: 0 @@ -34,13 +35,15 @@ requires: JSON: 0 List::MoreUtils: 0 MCE: 1.405 + Moo: 0 + Net::DNS: 0 Net::MAC: 0 - NetAddr::IP: 0 + NetAddr::IP: 4.059 Path::Class: 0 Plack: 1.0006 Plack::Middleware::Expires: 0 Role::Tiny: 0 - SNMP::Info: 2.11 + SNMP::Info: 3.00 SQL::Translator: 0 Socket6: 0 Starman: 0 diff --git a/Netdisco/Makefile.PL b/Netdisco/Makefile.PL index 414f22d0..a870c65a 100644 --- a/Netdisco/Makefile.PL +++ b/Netdisco/Makefile.PL @@ -4,6 +4,7 @@ name 'App-Netdisco'; license 'bsd'; all_from 'lib/App/Netdisco.pm'; +requires 'Algorithm::Cron' => 0; requires 'App::cpanminus' => 0; requires 'App::local::lib::helper' => 0; requires 'DBD::Pg' => 0; @@ -18,16 +19,18 @@ requires 'HTML::Entities' => 0; requires 'HTTP::Tiny' => 0; requires 'JSON' => 0; requires 'List::MoreUtils' => 0; +requires 'Moo' => 0; requires 'MCE' => 1.405; +requires 'Net::DNS' => 0; requires 'Net::MAC' => 0; -requires 'NetAddr::IP' => 0; +requires 'NetAddr::IP' => '4.059'; requires 'Path::Class' => 0; requires 'Plack' => 1.0006; requires 'Plack::Middleware::Expires' => 0; requires 'Role::Tiny' => 0; requires 'Socket6' => 0; requires 'Starman' => 0; -requires 'SNMP::Info' => '2.11'; +requires 'SNMP::Info' => '3.00'; requires 'SQL::Translator' => 0; requires 'Template' => 0; requires 'YAML' => 0; diff --git a/Netdisco/bin/netdisco-daemon-fg b/Netdisco/bin/netdisco-daemon-fg index 718b6e88..523c9f69 100755 --- a/Netdisco/bin/netdisco-daemon-fg +++ b/Netdisco/bin/netdisco-daemon-fg @@ -33,7 +33,7 @@ mkdir $tmp_dir if ! -d $tmp_dir; my $mce = MCE->new( spawn_delay => 0.15, - job_delay => 0.15, + job_delay => 1.15, tmp_dir => $tmp_dir, user_func => sub { $_[0]->worker_body }, on_post_exit => \&restart_worker, @@ -41,17 +41,23 @@ my $mce = MCE->new( )->run(); sub build_tasks_list { - my $tasks = [{ - max_workers => 1, - user_begin => worker_factory('Manager'), - }]; + # NB MCE does not like max_workers => 0 + my $tasks = []; set(daemon_pollers => 2) if !defined setting('daemon_pollers'); set(daemon_interactives => 2) if !defined setting('daemon_interactives'); - # XXX MCE does not like max_workers => 0 + push @$tasks, { + max_workers => 1, + user_begin => worker_factory('Manager'), + } if setting('daemon_pollers') or setting('daemon_interactives'); + + push @$tasks, { + max_workers => 1, + user_begin => worker_factory('Scheduler'), + } if setting('housekeeping'); push @$tasks, { max_workers => setting('daemon_pollers'), @@ -63,8 +69,13 @@ sub build_tasks_list { user_begin => worker_factory('Interactive'), } if setting('daemon_interactives'); - info sprintf "MCE will load %s tasks: 1 Manager, %s Poller, %s Interactive", - (1+ scalar @$tasks), (setting('daemon_pollers') || 0), (setting('daemon_interactives') || 0); + info sprintf "MCE will load %s tasks: %s Manager, %s Scheduler, %s Poller, %s Interactive", + (scalar @$tasks), + ((setting('daemon_pollers') or setting('daemon_interactives')) ? 1 : 0), + (setting('housekeeping') ? 1 : 0), + (setting('daemon_pollers') || 0), + (setting('daemon_interactives') || 0); + return $tasks; } diff --git a/Netdisco/bin/netdisco-do b/Netdisco/bin/netdisco-do new file mode 100755 index 00000000..f88ef910 --- /dev/null +++ b/Netdisco/bin/netdisco-do @@ -0,0 +1,108 @@ +#!/usr/bin/env perl + +use FindBin; +FindBin::again(); +use Path::Class 'dir'; + +BEGIN { + # stuff useful locations into @INC + unshift @INC, + dir($FindBin::RealBin)->parent->subdir('lib')->stringify, + dir($FindBin::RealBin, 'lib')->stringify; +} + +# for netdisco app config +use App::Netdisco; +use Dancer qw/:moose :script/; +use Dancer::Plugin::DBIC 'schema'; + +info "App::Netdisco version $App::Netdisco::VERSION loaded."; + +use Try::Tiny; +use Getopt::Long; +Getopt::Long::Configure ("bundling"); + +my ($device, $port, $extra, $debug); +my $result = GetOptions( + 'device|d=s' => \$device, + 'port|p=s' => \$port, + 'extra|e=s' => \$extra, + 'debug|D' => \$debug, +) or exit(1); + +# reconfigure logging to use console +my $CONFIG = config(); +$CONFIG->{logger} = 'console'; +$CONFIG->{log} = ($debug ? 'debug' : 'info'); + +Dancer::Logger->init('console', $CONFIG); + +# check requested action +my $action = shift @ARGV; +my $PERMITTED_ACTIONS = qr/(?:discover|discover_neighbors)/; + +if (!length $action) { + error 'error: missing action!'; + exit (1); +} + +if ($action !~ m/^$PERMITTED_ACTIONS$/) { + error sprintf 'error: netdisco-do cannot [%s]', $action; + exit (1); +} + +if (!length $device) { + error 'error: missing device!'; + exit (1); +} + +# create worker (placeholder object for the role methods) +{ + package MyWorker; + use Moo; + with 'App::Netdisco::Daemon::Worker::Poller::Discover'; +} +my $worker = MyWorker->new(); + +# static configuration for the in-memory local job queue +setting('plugins')->{DBIC}->{daemon} = { + dsn => 'dbi:SQLite:dbname=:memory:', + options => { + AutoCommit => 1, + RaiseError => 1, + sqlite_use_immediate_transaction => 1, + }, + schema_class => 'App::Netdisco::Daemon::DB', +}; +schema('daemon')->deploy; + +# what job are we asked to do? +my $job = schema('daemon')->resultset('Admin')->new_result({ + job => 0, + action => $action, + device => $device, + port => $port, + subaction => $extra, +}); + +# belt and braces check before we go ahead +if (not $worker->can( $action )) { + error sprintf 'error: %s is not a valid action for netdisco-do', $action; + exit (1); +} + +# do job +my ($status, $log); +try { + info sprintf '%s: started at %s', $action, scalar localtime; + ($status, $log) = $worker->$action($job); +} +catch { + $status = 'error'; + $log = "error running job: $_"; +}; + +info sprintf '%s: finished at %s', $action, scalar localtime; +info sprintf '%s: status %s: %s', $action, $status, $log; + +exit ($status eq 'done' ? 0 : 1); diff --git a/Netdisco/lib/App/Netdisco/DB.pm b/Netdisco/lib/App/Netdisco/DB.pm index 938bb7d0..db2129ad 100644 --- a/Netdisco/lib/App/Netdisco/DB.pm +++ b/Netdisco/lib/App/Netdisco/DB.pm @@ -1,9 +1,6 @@ use utf8; package App::Netdisco::DB; -# Created by DBIx::Class::Schema::Loader -# DO NOT MODIFY THE FIRST PART OF THIS FILE - use strict; use warnings; @@ -11,11 +8,7 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_namespaces; - -# Created by DBIx::Class::Schema::Loader v0.07015 @ 2012-01-07 14:20:02 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tQTf/oInVydRDsuIFLSU4A - -our $VERSION = 16; # schema version used for upgrades, keep as integer +our $VERSION = 17; # schema version used for upgrades, keep as integer use Path::Class; use File::Basename; @@ -27,5 +20,4 @@ our $schema_versions_dir = Path::Class::Dir->new($libpath) __PACKAGE__->load_components(qw/Schema::Versioned/); __PACKAGE__->upgrade_directory($schema_versions_dir); -# You can replace this text with custom code or comments, and it will be preserved on regeneration 1; diff --git a/Netdisco/lib/App/Netdisco/DB/Result/Device.pm b/Netdisco/lib/App/Netdisco/DB/Result/Device.pm index 3fd16244..19774496 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/Device.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/Device.pm @@ -110,6 +110,22 @@ Returns the set of ports on this Device. __PACKAGE__->has_many( ports => 'App::Netdisco::DB::Result::DevicePort', 'ip' ); +=head2 modules + +Returns the set chassis modules on this Device. + +=cut + +__PACKAGE__->has_many( modules => 'App::Netdisco::DB::Result::DeviceModule', 'ip' ); + +=head2 power_modules + +Returns the set of power modules on this Device. + +=cut + +__PACKAGE__->has_many( power_modules => 'App::Netdisco::DB::Result::DevicePower', 'ip' ); + =head2 port_vlans Returns the set of VLANs known to be configured on Ports on this Device, @@ -128,6 +144,40 @@ __PACKAGE__->has_many( # helper which assumes we've just RIGHT JOINed to Vlans table sub vlan { return (shift)->vlans->first } +=head2 wireless_ports + +Returns the set of wireless IDs known to be configured on Ports on this +Device. + +=cut + +__PACKAGE__->has_many( + wireless_ports => 'App::Netdisco::DB::Result::DevicePortWireless', + 'ip', { join_type => 'RIGHT' } +); + +=head2 ssids + +Returns the set of SSIDs known to be configured on Ports on this Device. + +=cut + +__PACKAGE__->has_many( + ssids => 'App::Netdisco::DB::Result::DevicePortSsid', + 'ip', { join_type => 'RIGHT' } +); + +=head2 powered_ports + +Returns the set of ports known to have PoE capability + +=cut + +__PACKAGE__->has_many( + powered_ports => 'App::Netdisco::DB::Result::DevicePortPower', + 'ip', { join_type => 'RIGHT' } +); + =head1 ADDITIONAL COLUMNS =head2 uptime_age diff --git a/Netdisco/lib/App/Netdisco/DB/Result/DeviceModule.pm b/Netdisco/lib/App/Netdisco/DB/Result/DeviceModule.pm index f9ccc2fb..d7db3bab 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/DeviceModule.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/DeviceModule.pm @@ -54,6 +54,15 @@ __PACKAGE__->set_primary_key("ip", "index"); # Created by DBIx::Class::Schema::Loader v0.07015 @ 2012-01-07 14:20:02 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:nuwxZBoiip9trdJFmgk3Fw +=head1 RELATIONSHIPS + +=head2 device + +Returns the entry from the C table on which this VLAN entry was discovered. + +=cut + +__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' ); # You can replace this text with custom code or comments, and it will be preserved on regeneration 1; diff --git a/Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm b/Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm index f718edf6..aad723cb 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm @@ -72,7 +72,19 @@ Returns the Device table entry to which the given Port is related. =cut -__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip'); +__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' ); + +=head2 vlans + +Returns the set of C entries associated with this Port. + +These will be both native and non-native (tagged). See also +C and C. + +=cut + +__PACKAGE__->has_many( vlans => 'App::Netdisco::DB::Result::DevicePortVlan', + { 'foreign.ip' => 'self.ip', 'foreign.port' => 'self.port' } ); =head2 nodes / active_nodes / nodes_with_age / active_nodes_with_age diff --git a/Netdisco/lib/App/Netdisco/DB/Result/DevicePortSsid.pm b/Netdisco/lib/App/Netdisco/DB/Result/DevicePortSsid.pm index b3c3d9b2..2155d3d1 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/DevicePortSsid.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/DevicePortSsid.pm @@ -26,6 +26,25 @@ __PACKAGE__->add_columns( # Created by DBIx::Class::Schema::Loader v0.07015 @ 2012-01-07 14:20:02 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zvgylKzUQtizJZCe1rEdUg +=head1 RELATIONSHIPS + +=head2 device + +Returns the entry from the C table which hosts this SSID. + +=cut + +__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' ); + +=head2 port + +Returns the entry from the C table which corresponds to this SSID. + +=cut + +__PACKAGE__->belongs_to( port => 'App::Netdisco::DB::Result::DevicePort', { + 'foreign.ip' => 'self.ip', 'foreign.port' => 'self.port', +}); # You can replace this text with custom code or comments, and it will be preserved on regeneration 1; diff --git a/Netdisco/lib/App/Netdisco/DB/Result/DevicePortWireless.pm b/Netdisco/lib/App/Netdisco/DB/Result/DevicePortWireless.pm index 6c7a6633..7cc6085b 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/DevicePortWireless.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/DevicePortWireless.pm @@ -24,6 +24,26 @@ __PACKAGE__->add_columns( # Created by DBIx::Class::Schema::Loader v0.07015 @ 2012-01-07 14:20:02 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:T5GmnCj/9BB7meiGZ3xN7g +=head1 RELATIONSHIPS + +=head2 device + +Returns the entry from the C table which hosts this wireless port. + +=cut + +__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' ); + +=head2 port + +Returns the entry from the C table which corresponds to this wireless +interface. + +=cut + +__PACKAGE__->belongs_to( port => 'App::Netdisco::DB::Result::DevicePort', { + 'foreign.ip' => 'self.ip', 'foreign.port' => 'self.port', +}); # You can replace this text with custom code or comments, and it will be preserved on regeneration 1; diff --git a/Netdisco/lib/App/Netdisco/DB/Result/DevicePower.pm b/Netdisco/lib/App/Netdisco/DB/Result/DevicePower.pm index 1a70858c..2f2cbbe5 100644 --- a/Netdisco/lib/App/Netdisco/DB/Result/DevicePower.pm +++ b/Netdisco/lib/App/Netdisco/DB/Result/DevicePower.pm @@ -25,6 +25,15 @@ __PACKAGE__->set_primary_key("ip", "module"); # Created by DBIx::Class::Schema::Loader v0.07015 @ 2012-01-07 14:20:02 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:awZRI/IH2VewzGlxISsr7w +=head1 RELATIONSHIPS + +=head2 device + +Returns the entry from the C table on which this power module was discovered. + +=cut + +__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' ); # You can replace this text with custom code or comments, and it will be preserved on regeneration 1; diff --git a/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-16-17-PostgreSQL.sql b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-16-17-PostgreSQL.sql new file mode 100644 index 00000000..6f768d16 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-16-17-PostgreSQL.sql @@ -0,0 +1,12 @@ + +BEGIN; + +CREATE UNIQUE INDEX jobs_queued ON admin ( + action, + coalesce(subaction, '_x_'), + coalesce(device, '255.255.255.255'), + coalesce(port, '_x_') +) WHERE status LIKE 'queued%'; + +COMMIT; + diff --git a/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-17-PostgreSQL.sql b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-17-PostgreSQL.sql new file mode 100644 index 00000000..67c21f06 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-17-PostgreSQL.sql @@ -0,0 +1,436 @@ +-- +-- Created by SQL::Translator::Producer::PostgreSQL +-- Created on Sun Mar 24 18:45:08 2013 +-- +-- +-- Table: admin. +-- +DROP TABLE "admin" CASCADE; +CREATE TABLE "admin" ( + "job" serial NOT NULL, + "entered" timestamp DEFAULT current_timestamp, + "started" timestamp, + "finished" timestamp, + "device" inet, + "port" text, + "action" text, + "subaction" text, + "status" text, + "username" text, + "userip" inet, + "log" text, + "debug" boolean, + PRIMARY KEY ("job"), +); + +CREATE UNIQUE INDEX jobs_queued ON admin ( + action, + coalesce(subaction, '_x_'), + coalesce(device, '255.255.255.255'), + coalesce(port, '_x_') +) WHERE status LIKE 'queued%'; + +-- +-- Table: device. +-- +DROP TABLE "device" CASCADE; +CREATE TABLE "device" ( + "ip" inet NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "dns" text, + "description" text, + "uptime" bigint, + "contact" text, + "name" text, + "location" text, + "layers" character varying(8), + "ports" integer, + "mac" macaddr, + "serial" text, + "model" text, + "ps1_type" text, + "ps2_type" text, + "ps1_status" text, + "ps2_status" text, + "fan" text, + "slots" integer, + "vendor" text, + "os" text, + "os_ver" text, + "log" text, + "snmp_ver" integer, + "snmp_comm" text, + "snmp_class" text, + "vtp_domain" text, + "last_discover" timestamp, + "last_macsuck" timestamp, + "last_arpnip" timestamp, + PRIMARY KEY ("ip") +); + +-- +-- Table: device_module. +-- +DROP TABLE "device_module" CASCADE; +CREATE TABLE "device_module" ( + "ip" inet NOT NULL, + "index" integer NOT NULL, + "description" text, + "type" text, + "parent" integer, + "name" text, + "class" text, + "pos" integer, + "hw_ver" text, + "fw_ver" text, + "sw_ver" text, + "serial" text, + "model" text, + "fru" boolean, + "creation" timestamp DEFAULT current_timestamp, + "last_discover" timestamp, + PRIMARY KEY ("ip", "index") +); + +-- +-- Table: device_port_log. +-- +DROP TABLE "device_port_log" CASCADE; +CREATE TABLE "device_port_log" ( + "id" serial NOT NULL, + "ip" inet, + "port" text, + "reason" text, + "log" text, + "username" text, + "userip" inet, + "action" text, + "creation" timestamp DEFAULT current_timestamp +); + +-- +-- Table: device_port_ssid. +-- +DROP TABLE "device_port_ssid" CASCADE; +CREATE TABLE "device_port_ssid" ( + "ip" inet, + "port" text, + "ssid" text, + "broadcast" boolean, + "bssid" macaddr +); + +-- +-- Table: device_port_wireless. +-- +DROP TABLE "device_port_wireless" CASCADE; +CREATE TABLE "device_port_wireless" ( + "ip" inet, + "port" text, + "channel" integer, + "power" integer +); + +-- +-- Table: device_power. +-- +DROP TABLE "device_power" CASCADE; +CREATE TABLE "device_power" ( + "ip" inet NOT NULL, + "module" integer NOT NULL, + "power" integer, + "status" text, + PRIMARY KEY ("ip", "module") +); + +-- +-- Table: device_route. +-- +DROP TABLE "device_route" CASCADE; +CREATE TABLE "device_route" ( + "ip" inet NOT NULL, + "network" cidr NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "dest" inet NOT NULL, + "last_discover" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("ip", "network", "dest") +); + +-- +-- Table: log. +-- +DROP TABLE "log" CASCADE; +CREATE TABLE "log" ( + "id" serial NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "class" text, + "entry" text, + "logfile" text +); + +-- +-- Table: node_ip. +-- +DROP TABLE "node_ip" CASCADE; +CREATE TABLE "node_ip" ( + "mac" macaddr NOT NULL, + "ip" inet NOT NULL, + "dns" text, + "active" boolean, + "time_first" timestamp DEFAULT current_timestamp, + "time_last" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("mac", "ip") +); + +-- +-- Table: node_monitor. +-- +DROP TABLE "node_monitor" CASCADE; +CREATE TABLE "node_monitor" ( + "mac" macaddr NOT NULL, + "active" boolean, + "why" text, + "cc" text, + "date" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("mac") +); + +-- +-- Table: node_nbt. +-- +DROP TABLE "node_nbt" CASCADE; +CREATE TABLE "node_nbt" ( + "mac" macaddr NOT NULL, + "ip" inet, + "nbname" text, + "domain" text, + "server" boolean, + "nbuser" text, + "active" boolean, + "time_first" timestamp DEFAULT current_timestamp, + "time_last" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("mac") +); + +-- +-- Table: node_wireless. +-- +DROP TABLE "node_wireless" CASCADE; +CREATE TABLE "node_wireless" ( + "mac" macaddr NOT NULL, + "uptime" integer, + "maxrate" integer, + "txrate" integer, + "sigstrength" integer, + "sigqual" integer, + "rxpkt" integer, + "txpkt" integer, + "rxbyte" bigint, + "txbyte" bigint, + "time_last" timestamp DEFAULT current_timestamp, + "ssid" text DEFAULT '' NOT NULL, + PRIMARY KEY ("mac", "ssid") +); + +-- +-- Table: oui. +-- +DROP TABLE "oui" CASCADE; +CREATE TABLE "oui" ( + "oui" character varying(8) NOT NULL, + "company" text, + PRIMARY KEY ("oui") +); + +-- +-- Table: process. +-- +DROP TABLE "process" CASCADE; +CREATE TABLE "process" ( + "controller" integer NOT NULL, + "device" inet NOT NULL, + "action" text NOT NULL, + "status" text, + "count" integer, + "creation" timestamp DEFAULT current_timestamp +); + +-- +-- Table: sessions. +-- +DROP TABLE "sessions" CASCADE; +CREATE TABLE "sessions" ( + "id" character(32) NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "a_session" text, + PRIMARY KEY ("id") +); + +-- +-- Table: subnets. +-- +DROP TABLE "subnets" CASCADE; +CREATE TABLE "subnets" ( + "net" cidr NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "last_discover" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("net") +); + +-- +-- Table: topology. +-- +DROP TABLE "topology" CASCADE; +CREATE TABLE "topology" ( + "dev1" inet NOT NULL, + "port1" text NOT NULL, + "dev2" inet NOT NULL, + "port2" text NOT NULL +); + +-- +-- Table: user_log. +-- +DROP TABLE "user_log" CASCADE; +CREATE TABLE "user_log" ( + "entry" serial NOT NULL, + "username" character varying(50), + "userip" inet, + "event" text, + "details" text, + "creation" timestamp DEFAULT current_timestamp +); + +-- +-- Table: users. +-- +DROP TABLE "users" CASCADE; +CREATE TABLE "users" ( + "username" character varying(50) NOT NULL, + "password" text, + "creation" timestamp DEFAULT current_timestamp, + "last_on" timestamp, + "port_control" boolean DEFAULT false, + "ldap" boolean DEFAULT false, + "admin" boolean DEFAULT false, + "fullname" text, + "note" text, + PRIMARY KEY ("username") +); + +-- +-- Table: device_vlan. +-- +DROP TABLE "device_vlan" CASCADE; +CREATE TABLE "device_vlan" ( + "ip" inet NOT NULL, + "vlan" integer NOT NULL, + "description" text, + "creation" timestamp DEFAULT current_timestamp, + "last_discover" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("ip", "vlan") +); +CREATE INDEX "device_vlan_idx_ip" on "device_vlan" ("ip"); + +-- +-- Table: device_ip. +-- +DROP TABLE "device_ip" CASCADE; +CREATE TABLE "device_ip" ( + "ip" inet NOT NULL, + "alias" inet NOT NULL, + "subnet" cidr, + "port" text, + "dns" text, + "creation" timestamp DEFAULT current_timestamp, + PRIMARY KEY ("ip", "alias"), + CONSTRAINT "device_ip_alias" UNIQUE ("alias") +); +CREATE INDEX "device_ip_idx_ip" on "device_ip" ("ip"); +CREATE INDEX "device_ip_idx_ip_port" on "device_ip" ("ip", "port"); + +-- +-- Table: device_port. +-- +DROP TABLE "device_port" CASCADE; +CREATE TABLE "device_port" ( + "ip" inet NOT NULL, + "port" text NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "descr" text, + "up" text, + "up_admin" text, + "type" text, + "duplex" text, + "duplex_admin" text, + "speed" text, + "name" text, + "mac" macaddr, + "mtu" integer, + "stp" text, + "remote_ip" inet, + "remote_port" text, + "remote_type" text, + "remote_id" text, + "vlan" text, + "pvid" integer, + "lastchange" bigint, + PRIMARY KEY ("port", "ip") +); +CREATE INDEX "device_port_idx_ip" on "device_port" ("ip"); +CREATE INDEX "device_port_idx_remote_ip" on "device_port" ("remote_ip"); + +-- +-- Table: device_port_power. +-- +DROP TABLE "device_port_power" CASCADE; +CREATE TABLE "device_port_power" ( + "ip" inet NOT NULL, + "port" text NOT NULL, + "module" integer, + "admin" text, + "status" text, + "class" text, + "power" integer, + PRIMARY KEY ("port", "ip") +); +CREATE INDEX "device_port_power_idx_ip_port" on "device_port_power" ("ip", "port"); + +-- +-- Table: device_port_vlan. +-- +DROP TABLE "device_port_vlan" CASCADE; +CREATE TABLE "device_port_vlan" ( + "ip" inet NOT NULL, + "port" text NOT NULL, + "vlan" integer NOT NULL, + "native" boolean DEFAULT false NOT NULL, + "creation" timestamp DEFAULT current_timestamp, + "last_discover" timestamp DEFAULT current_timestamp, + "vlantype" text, + PRIMARY KEY ("ip", "port", "vlan") +); +CREATE INDEX "device_port_vlan_idx_ip" on "device_port_vlan" ("ip"); +CREATE INDEX "device_port_vlan_idx_ip_port" on "device_port_vlan" ("ip", "port"); +CREATE INDEX "device_port_vlan_idx_ip_vlan" on "device_port_vlan" ("ip", "vlan"); + +-- +-- Table: node. +-- +DROP TABLE "node" CASCADE; +CREATE TABLE "node" ( + "mac" macaddr NOT NULL, + "switch" inet NOT NULL, + "port" text NOT NULL, + "active" boolean, + "oui" character varying(8), + "time_first" timestamp DEFAULT current_timestamp, + "time_recent" timestamp DEFAULT current_timestamp, + "time_last" timestamp DEFAULT current_timestamp, + "vlan" text DEFAULT '0' NOT NULL, + PRIMARY KEY ("mac", "switch", "port", "vlan") +); +CREATE INDEX "node_idx_switch" on "node" ("switch"); +CREATE INDEX "node_idx_switch_port" on "node" ("switch", "port"); +CREATE INDEX "node_idx_oui" on "node" ("oui"); + diff --git a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm index 056a9c48..95cbb30d 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Queue.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Queue.pm @@ -33,11 +33,13 @@ sub capacity_for { debug "checking local capacity for action $action"; my $action_map = { - Interactive => [qw/location contact portcontrol portname vlan power/] + Poller => [qw/refresh discover discoverall discover_neighbors/], + Interactive => [qw/location contact portcontrol portname vlan power/], }; my $role_map = { - map {$_ => 'Interactive'} @{ $action_map->{Interactive} } + (map {$_ => 'Poller'} @{ $action_map->{Poller} }), + (map {$_ => 'Interactive'} @{ $action_map->{Interactive} }) }; my $setting_map = { diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm index 9a6a8312..37819c3b 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm @@ -35,7 +35,8 @@ sub worker_body { my ($status, $log); try { $job->started(scalar localtime); - info sprintf "int (%s): starting job %s at %s", $wid, $jid, $job->started; + info sprintf "int (%s): starting %s job(%s) at %s", + $wid, $target, $jid, $job->started; ($status, $log) = $self->$target($job); } catch { @@ -55,8 +56,8 @@ sub worker_body { sub close_job { my ($self, $job, $status, $log) = @_; my $now = scalar localtime; - info sprintf "int (%s): wrapping up job %s - status %s at %s", - $self->wid, $job->job, $status, $now; + info sprintf "int (%s): wrapping up set_%s job(%s) - status %s at %s", + $self->wid, $job->action, $job->job, $status, $now; try { schema('netdisco')->resultset('Admin') diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm index 3a411f17..70362313 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm @@ -1,6 +1,7 @@ package App::Netdisco::Daemon::Worker::Interactive::DeviceActions; -use App::Netdisco::Util::Connect qw/snmp_connect get_device/; +use App::Netdisco::Util::SNMP ':all'; +use App::Netdisco::Util::Device 'get_device'; use App::Netdisco::Daemon::Worker::Interactive::Util ':all'; use Role::Tiny; @@ -21,14 +22,14 @@ sub _set_device_generic { $data ||= ''; # snmp connect using rw community - my $info = snmp_connect($ip) - or return error("Failed to connect to device [$ip] to update $slot"); + my $info = snmp_connect_rw($ip) + or return job_error("Failed to connect to device [$ip] to update $slot"); my $method = 'set_'. $slot; my $rv = $info->$method($data); if (!defined $rv) { - return error(sprintf 'Failed to set %s on [%s]: %s', + return job_error(sprintf 'Failed to set %s on [%s]: %s', $slot, $ip, ($info->error || '')); } @@ -36,14 +37,14 @@ sub _set_device_generic { $info->clear_cache; my $new_data = ($info->$slot || ''); if ($new_data ne $data) { - return error("Verify of $slot update failed on [$ip]: $new_data"); + return job_error("Verify of $slot update failed on [$ip]: $new_data"); } # update netdisco DB my $device = get_device($ip); $device->update({$slot => $data}); - return done("Updated $slot on [$ip] to [$data]"); + return job_done("Updated $slot on [$ip] to [$data]"); } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm index 38e44f78..1e0c0e48 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm @@ -1,7 +1,7 @@ package App::Netdisco::Daemon::Worker::Interactive::PortActions; -use App::Netdisco::Util::Connect ':all'; -use App::Netdisco::Util::Permissions ':all'; +use App::Netdisco::Util::SNMP ':all'; +use App::Netdisco::Util::Port ':all'; use App::Netdisco::Daemon::Worker::Interactive::Util ':all'; use Role::Tiny; @@ -16,11 +16,11 @@ sub set_portcontrol { my ($self, $job) = @_; my $port = get_port($job->device, $job->port) - or return error(sprintf "Unknown port name [%s] on device [%s]", + or return job_error(sprintf "Unknown port name [%s] on device [%s]", $job->port, $job->device); my $reconfig_check = port_reconfig_check($port); - return error("Cannot alter port: $reconfig_check") + return job_error("Cannot alter port: $reconfig_check") if length $reconfig_check; return _set_port_generic($job, 'up_admin'); @@ -30,15 +30,15 @@ sub set_vlan { my ($self, $job) = @_; my $port = get_port($job->device, $job->port) - or return error(sprintf "Unknown port name [%s] on device [%s]", + or return job_error(sprintf "Unknown port name [%s] on device [%s]", $job->port, $job->device); my $port_reconfig_check = port_reconfig_check($port); - return error("Cannot alter port: $port_reconfig_check") + return job_error("Cannot alter port: $port_reconfig_check") if length $port_reconfig_check; my $vlan_reconfig_check = vlan_reconfig_check($port); - return error("Cannot alter vlan: $vlan_reconfig_check") + return job_error("Cannot alter vlan: $vlan_reconfig_check") if length $vlan_reconfig_check; return _set_port_generic($job, 'vlan'); @@ -53,20 +53,20 @@ sub _set_port_generic { (my $data = $job->subaction) =~ s/-\w+//; my $port = get_port($ip, $pn) - or return error("Unknown port name [$pn] on device [$ip]"); + or return job_error("Unknown port name [$pn] on device [$ip]"); # snmp connect using rw community - my $info = snmp_connect($ip) - or return error("Failed to connect to device [$ip] to control port"); + my $info = snmp_connect_rw($ip) + or return job_error("Failed to connect to device [$ip] to control port"); my $iid = get_iid($info, $port) - or return error("Failed to get port ID for [$pn] from [$ip]"); + or return job_error("Failed to get port ID for [$pn] from [$ip]"); my $method = 'set_i_'. $slot; my $rv = $info->$method($data, $iid); if (!defined $rv) { - return error(sprintf 'Failed to set [%s] %s to [%s] on [%s]: %s', + return job_error(sprintf 'Failed to set [%s] %s to [%s] on [%s]: %s', $pn, $slot, $data, $ip, ($info->error || '')); } @@ -75,27 +75,27 @@ sub _set_port_generic { my $check_method = 'i_'. $slot; my $state = ($info->$check_method($iid) || ''); if (ref {} ne ref $state or $state->{$iid} ne $data) { - return error("Verify of [$pn] $slot failed on [$ip]"); + return job_error("Verify of [$pn] $slot failed on [$ip]"); } # update netdisco DB $port->update({$column => $data}); - return done("Updated [$pn] $slot status on [$ip] to [$data]"); + return job_done("Updated [$pn] $slot status on [$ip] to [$data]"); } sub set_power { my ($self, $job) = @_; my $port = get_port($job->device, $job->port) - or return error(sprintf "Unknown port name [%s] on device [%s]", + or return job_error(sprintf "Unknown port name [%s] on device [%s]", $job->port, $job->device); - return error("No PoE service on port [%s] on device [%s]") + return job_error("No PoE service on port [%s] on device [%s]") unless $port->power; my $reconfig_check = port_reconfig_check($port); - return error("Cannot alter port: $reconfig_check") + return job_error("Cannot alter port: $reconfig_check") if length $reconfig_check; @@ -104,16 +104,16 @@ sub set_power { (my $data = $job->subaction) =~ s/-\w+//; # snmp connect using rw community - my $info = snmp_connect($ip) - or return error("Failed to connect to device [$ip] to control port"); + my $info = snmp_connect_rw($ip) + or return job_error("Failed to connect to device [$ip] to control port"); my $powerid = get_powerid($info, $port) - or return error("Failed to get power ID for [$pn] from [$ip]"); + or return job_error("Failed to get power ID for [$pn] from [$ip]"); my $rv = $info->set_peth_port_admin($data, $powerid); if (!defined $rv) { - return error(sprintf 'Failed to set [%s] power to [%s] on [%s]: %s', + return job_error(sprintf 'Failed to set [%s] power to [%s] on [%s]: %s', $pn, $data, $ip, ($info->error || '')); } @@ -121,7 +121,7 @@ sub set_power { $info->clear_cache; my $state = ($info->peth_port_admin($powerid) || ''); if (ref {} ne ref $state or $state->{$powerid} ne $data) { - return error("Verify of [$pn] power failed on [$ip]"); + return job_error("Verify of [$pn] power failed on [$ip]"); } # update netdisco DB @@ -130,7 +130,7 @@ sub set_power { status => ($data eq 'false' ? 'disabled' : 'searching'), }); - return done("Updated [$pn] power status on [$ip] to [$data]"); + return job_done("Updated [$pn] power status on [$ip] to [$data]"); } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/Util.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/Util.pm index b6844bbf..4853ad3f 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/Util.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive/Util.pm @@ -4,12 +4,10 @@ package App::Netdisco::Daemon::Worker::Interactive::Util; use base 'Exporter'; our @EXPORT = (); -our @EXPORT_OK = qw/ done error /; -our %EXPORT_TAGS = ( - all => [qw/ done error /], -); +our @EXPORT_OK = qw/ job_done job_error /; +our %EXPORT_TAGS = (all => \@EXPORT_OK); -sub done { return ('done', shift) } -sub error { return ('error', shift) } +sub job_done { return ('done', shift) } +sub job_error { return ('error', shift) } 1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm index abd8ac8a..1216b6ad 100644 --- a/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm @@ -3,7 +3,7 @@ package App::Netdisco::Daemon::Worker::Manager; use Dancer qw/:moose :syntax :script/; use Dancer::Plugin::DBIC 'schema'; -use App::Netdisco::Util::DeviceProperties 'is_discoverable'; +use App::Netdisco::Util::Device 'is_discoverable'; use Net::Domain 'hostfqdn'; use Try::Tiny; @@ -13,8 +13,10 @@ use namespace::clean; my $fqdn = hostfqdn || 'localhost'; my $role_map = { - map {$_ => 'Interactive'} - qw/location contact portcontrol portname vlan power/ + (map {$_ => 'Poller'} + qw/refresh discover discoverall discover_neighbors/), + (map {$_ => 'Interactive'} + qw/location contact portcontrol portname vlan power/) }; sub worker_begin { @@ -40,7 +42,8 @@ sub worker_begin { sub worker_body { my $self = shift; my $wid = $self->wid; - my $num_slots = $self->do('num_workers'); + my $num_slots = $self->do('num_workers') + or return debug "mgr ($wid): this node has no workers... quitting manager"; # get some pending jobs my $rs = schema('netdisco')->resultset('Admin') @@ -56,11 +59,11 @@ sub worker_body { # filter for discover_* next unless is_discoverable($job->device); - info sprintf "mgr (%s): job %s is discoverable", $wid, $jid; + debug sprintf "mgr (%s): job %s is discoverable", $wid, $jid; # check for available local capacity next unless $self->do('capacity_for', $job->action); - info sprintf "mgr (%s): processing node has capacity for job %s (%s)", + debug sprintf "mgr (%s): processing node has capacity for job %s (%s)", $wid, $jid, $job->action; # mark job as running diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm new file mode 100644 index 00000000..4eefde88 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm @@ -0,0 +1,75 @@ +package App::Netdisco::Daemon::Worker::Poller; + +use Dancer qw/:moose :syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use Try::Tiny; + +use Role::Tiny; +use namespace::clean; + +# add dispatch methods for poller tasks +with 'App::Netdisco::Daemon::Worker::Poller::Discover'; + +sub worker_body { + my $self = shift; + my $wid = $self->wid; + + while (1) { + debug "poll ($wid): asking for a job"; + my $jobs = $self->do('take_jobs', $self->wid, 'Poller'); + + foreach my $candidate (@$jobs) { + # create a row object so we can use column accessors + # use the local db schema in case it is accidentally 'stored' + # (will throw an exception) + my $job = schema('daemon')->resultset('Admin') + ->new_result($candidate); + my $jid = $job->job; + my $target = $job->action; + + next unless $self->can($target); + debug "poll ($wid): can ${target}() for job $jid"; + + # do job + my ($status, $log); + try { + $job->started(scalar localtime); + info sprintf "poll (%s): starting %s job(%s) at %s", + $wid, $target, $jid, $job->started; + ($status, $log) = $self->$target($job); + } + catch { + $status = 'error'; + $log = "error running job: $_"; + $self->sendto('stderr', $log ."\n"); + }; + + $self->close_job($job, $status, $log); + } + + debug "poll ($wid): sleeping now..."; + sleep( setting('daemon_sleep_time') || 5 ); + } +} + +sub close_job { + my ($self, $job, $status, $log) = @_; + my $now = scalar localtime; + info sprintf "poll (%s): wrapping up %s job(%s) - status %s at %s", + $self->wid, $job->action, $job->job, $status, $now; + + try { + schema('netdisco')->resultset('Admin') + ->find($job->job) + ->update({ + status => $status, + log => $log, + started => $job->started, + finished => $now, + }); + } + catch { $self->sendto('stderr', "error closing job: $_\n") }; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Discover.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Discover.pm new file mode 100644 index 00000000..abfeff9a --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Discover.pm @@ -0,0 +1,88 @@ +package App::Netdisco::Daemon::Worker::Poller::Discover; + +use Dancer qw/:moose :syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::SNMP 'snmp_connect'; +use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::DiscoverAndStore ':all'; +use App::Netdisco::Daemon::Worker::Interactive::Util ':all'; + +use NetAddr::IP::Lite ':lower'; + +use Role::Tiny; +use namespace::clean; + +# queue a discover job for all devices known to Netdisco +sub refresh { + my ($self, $job) = @_; + + my $devices = schema('netdisco')->resultset('Device')->get_column('ip'); + + schema('netdisco')->resultset('Admin')->populate([ + map {{ + device => $_, + action => 'discover', + status => 'queued', + }} ($devices->all) + ]); + + return job_done("Queued discover job for all devices"); +} + +sub discover { + my ($self, $job) = @_; + + my $host = NetAddr::IP::Lite->new($job->device); + my $device = get_device($host->addr); + my $snmp = snmp_connect($device); + + if (!defined $snmp) { + return job_error("discover failed: could not SNMP connect to $host"); + } + + store_device($device, $snmp); + store_interfaces($device, $snmp); + store_wireless($device, $snmp); + store_vlans($device, $snmp); + store_power($device, $snmp); + store_modules($device, $snmp); + + return job_done("Ended discover for $host"); +} + +# run find_neighbors on all known devices, and run discover on any +# newly found devices. +sub discoverall { + my ($self, $job) = @_; + + my $devices = schema('netdisco')->resultset('Device')->get_column('ip'); + + schema('netdisco')->resultset('Admin')->populate([ + map {{ + device => $_, + action => 'discover_neighbors', + status => 'queued', + }} ($devices->all) + ]); + + return job_done("Queued discover_neighbors job for all devices"); +} + +sub discover_neighbors { + my ($self, $job) = @_; + + my $host = NetAddr::IP::Lite->new($job->device); + my $device = get_device($host->addr); + my $snmp = snmp_connect($device); + + if (!defined $snmp) { + return job_error("discover_neighbors failed: could not SNMP connect to $host"); + } + + find_neighbors($device, $snmp); + + return job_done("Ended find_neighbors for $host"); +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm new file mode 100644 index 00000000..98eabb43 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Daemon/Worker/Scheduler.pm @@ -0,0 +1,92 @@ +package App::Netdisco::Daemon::Worker::Scheduler; + +use Dancer qw/:moose :syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use Algorithm::Cron; +use Try::Tiny; + +use Role::Tiny; +use namespace::clean; + +my $jobactions = { + map {$_ => undef} qw/ + / +# saveconfigs +# discoverall +# refresh +# macwalk +# arpwalk +# nbtwalk +# backup +}; + +sub worker_begin { + my $self = shift; + my $wid = $self->wid; + debug "entering Scheduler ($wid) worker_begin()"; + + foreach my $a (keys %$jobactions) { + next unless setting('housekeeping') + and exists setting('housekeeping')->{$a}; + my $config = setting('housekeeping')->{$a}; + + # accept either single crontab format, or individual time fields + my $cron = Algorithm::Cron->new( + base => 'local', + %{ + (ref {} eq ref $config->{when}) + ? $config->{when} + : {crontab => $config->{when}} + } + ); + + $jobactions->{$a} = $config; + $jobactions->{$a}->{when} = $cron; + } +} + +sub worker_body { + my $self = shift; + my $wid = $self->wid; + + while (1) { + # sleep until some point in the next minute + my $naptime = 60 - (time % 60) + int(rand(45)); + debug "sched ($wid): sleeping for $naptime seconds"; + sleep $naptime; + + # NB next_time() returns the next *after* win_start + my $win_start = time - (time % 60) - 1; + my $win_end = $win_start + 60; + + # if any job is due, add it to the queue + foreach my $a (keys %$jobactions) { + next unless defined $jobactions->{$a}; + my $sched = $jobactions->{$a}; + + # next occurence of job must be in this minute's window + debug sprintf "sched ($wid): $a: win_start: %s, win_end: %s, next: %s", + $win_start, $win_end, $sched->{when}->next_time($win_start); + next unless $sched->{when}->next_time($win_start) <= $win_end; + + # queue it! + # due to a table constraint, this will (intentionally) fail if a + # similar job is already queued. + try { + debug "sched ($wid): queueing $a job"; + schema('netdisco')->resultset('Admin')->create({ + action => $a, + device => ($sched->{device} || undef), + subaction => ($sched->{extra} || undef), + status => 'queued', + }); + } + catch { + debug "sched ($wid): action $a was not queued (dupe?)"; + }; + } + } +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Manual/Developing.pod b/Netdisco/lib/App/Netdisco/Manual/Developing.pod index b18aa479..524af458 100644 --- a/Netdisco/lib/App/Netdisco/Manual/Developing.pod +++ b/Netdisco/lib/App/Netdisco/Manual/Developing.pod @@ -424,7 +424,7 @@ each). The daemon obviously needs to use L for device control. All the code for this has been factored out into the L namespace. -The L package provides for the creation of +The L package provides for the creation of SNMP::Info objects along with connection tests. So far, SNMPv3 is not supported. To enable trace logging of the SNMP::Info object simply set the C environment variable to a true value. The Connect library also diff --git a/Netdisco/lib/App/Netdisco/Util/Connect.pm b/Netdisco/lib/App/Netdisco/Util/Connect.pm deleted file mode 100644 index 09ffda61..00000000 --- a/Netdisco/lib/App/Netdisco/Util/Connect.pm +++ /dev/null @@ -1,153 +0,0 @@ -package App::Netdisco::Util::Connect; - -use Dancer qw/:syntax :script/; -use Dancer::Plugin::DBIC 'schema'; - -use SNMP::Info; -use Try::Tiny; -use Path::Class 'dir'; - -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ - get_device get_port get_iid get_powerid snmp_connect -/; -our %EXPORT_TAGS = ( - all => [qw/ - get_device get_port get_iid get_powerid snmp_connect - /], -); - -=head1 App::Netdisco::Util::Connect - -A set of helper subroutines to support parts of the Netdisco application. - -There are no default exports, however the C<:all> tag will export all -subroutines. - -=head2 get_device( $ip ) - -Given an IP address, returns a L object for the Device in -the Netdisco database. The IP can be for any interface on the device. - -Returns C if the device or interface IP is not known to Netdisco. - -=cut - -sub get_device { - my $ip = shift; - - my $alias = schema('netdisco')->resultset('DeviceIp') - ->search({alias => $ip})->first; - return if not eval { $alias->ip }; - - return schema('netdisco')->resultset('Device') - ->find({ip => $alias->ip}); -} - -=head2 get_port( $device, $portname ) - -=cut - -sub get_port { - my ($device, $portname) = @_; - - # accept either ip or dbic object - $device = get_device($device) - if not ref $device; - - my $port = schema('netdisco')->resultset('DevicePort') - ->find({ip => $device->ip, port => $portname}); - - return $port; -} - -=head2 get_iid( $info, $port ) - -=cut - -sub get_iid { - my ($info, $port) = @_; - - # accept either port name or dbic object - $port = $port->port if ref $port; - - my $interfaces = $info->interfaces; - my %rev_if = reverse %$interfaces; - my $iid = $rev_if{$port}; - - return $iid; -} - -=head2 get_powerid( $info, $port ) - -=cut - -sub get_powerid { - my ($info, $port) = @_; - - # accept either port name or dbic object - $port = $port->port if ref $port; - - my $iid = get_iid($info, $port) - or return undef; - - my $p_interfaces = $info->peth_port_ifindex; - my %rev_p_if = reverse %$p_interfaces; - my $powerid = $rev_p_if{$iid}; - - return $powerid; -} - -=head2 snmp_connect( $ip ) - -Given an IP address, returns an L instance configured for and -connected to that device. The IP can be any on the device, and the management -interface will be connected to. - -Returns C if the connection fails. - -=cut - -sub snmp_connect { - my $ip = shift; - - # get device details from db - my $device = get_device($ip) - or return (); - - # TODO: really only supporing v2c at the moment - my %snmp_args = ( - DestHost => $device->ip, - Version => ($device->snmp_ver || setting('snmpver') || 2), - Retries => (setting('snmpretries') || 2), - Timeout => (setting('snmptimeout') || 1000000), - MibDirs => [ _build_mibdirs() ], - AutoSpecify => 1, - IgnoreNetSNMPConf => 1, - Debug => ($ENV{INFO_TRACE} || 0), - ); - - my $info = undef; - my $last_comm = 0; - COMMUNITY: foreach my $c (@{ setting('community_rw') || []}) { - try { - $info = SNMP::Info->new(%snmp_args, Community => $c); - ++$last_comm if ( - $info - and (not defined $info->error) - and length $info->uptime - ); - }; - last COMMUNITY if $last_comm; - } - - return $info; -} - -sub _build_mibdirs { - return map { dir(setting('mibhome'), $_) } - @{ setting('mibdirs') || [] }; -} - -1; diff --git a/Netdisco/lib/App/Netdisco/Util/DNS.pm b/Netdisco/lib/App/Netdisco/Util/DNS.pm new file mode 100644 index 00000000..5e15dd1b --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Util/DNS.pm @@ -0,0 +1,53 @@ +package App::Netdisco::Util::DNS; + +use strict; +use warnings FATAL => 'all'; + +use Net::DNS; + +use base 'Exporter'; +our @EXPORT = (); +our @EXPORT_OK = qw/ + hostname_from_ip +/; +our %EXPORT_TAGS = (all => \@EXPORT_OK); + +=head1 NAME + +App::Netdisco::Util::DNS + +=head1 DESCRIPTION + +A set of helper subroutines to support parts of the Netdisco application. + +There are no default exports, however the C<:all> tag will export all +subroutines. + +=head1 EXPORT_OK + +=head2 hostname_from_ip( $ip ) + +Given an IP address (either IPv4 or IPv6), return the canonical hostname. + +Returns C if no PTR record exists for the IP. + +=cut + +sub hostname_from_ip { + my $ip = shift; + + my $res = Net::DNS::Resolver->new; + my $query = $res->search($ip); + + if ($query) { + foreach my $rr ($query->answer) { + next unless $rr->type eq "PTR"; + return $rr->ptrdname; + } + } + + return undef; +} + +1; + diff --git a/Netdisco/lib/App/Netdisco/Util/DeviceProperties.pm b/Netdisco/lib/App/Netdisco/Util/Device.pm similarity index 55% rename from Netdisco/lib/App/Netdisco/Util/DeviceProperties.pm rename to Netdisco/lib/App/Netdisco/Util/Device.pm index b696dd17..eeba5f2e 100644 --- a/Netdisco/lib/App/Netdisco/Util/DeviceProperties.pm +++ b/Netdisco/lib/App/Netdisco/Util/Device.pm @@ -1,4 +1,4 @@ -package App::Netdisco::Util::DeviceProperties; +package App::Netdisco::Util::Device; use Dancer qw/:syntax :script/; use Dancer::Plugin::DBIC 'schema'; @@ -8,27 +8,56 @@ use NetAddr::IP::Lite ':lower'; use base 'Exporter'; our @EXPORT = (); our @EXPORT_OK = qw/ + get_device is_discoverable - is_vlan_interface port_has_phone /; -our %EXPORT_TAGS = ( - all => [qw/ - is_discoverable - is_vlan_interface port_has_phone - /], -); +our %EXPORT_TAGS = (all => \@EXPORT_OK); -=head1 App::Netdisco::Util::DeviceProperties; +=head1 NAME + +App::Netdisco::Util::Device + +=head1 DESCRIPTION A set of helper subroutines to support parts of the Netdisco application. There are no default exports, however the C<:all> tag will export all subroutines. +=head1 EXPORT_OK + +=head2 get_device( $ip ) + +Given an IP address, returns a L object for the Device in +the Netdisco database. The IP can be for any interface on the device. + +If for any reason C<$ip> is already a C Device object, then it is +simply returned. + +If the device or interface IP is not known to Netdisco a new Device object is +created for the IP, and returned. This object is in-memory only and not yet +stored to the database. + +=cut + +sub get_device { + my $ip = shift; + + # naive check for existing DBIC object + return $ip if ref $ip; + + my $alias = schema('netdisco')->resultset('DeviceIp') + ->search({alias => $ip})->first; + $ip = $alias->ip if defined $alias; + + return schema('netdisco')->resultset('Device') + ->find_or_new({ip => $ip}); +} + =head2 is_discoverable( $ip ) -Given an IP address, returns C if Netdisco on this host is permitted to -discover its configuration by the local configuration. +Given an IP address, returns C if Netdisco on this host is permitted by +the local configuration to discover the device. The configuration items C and C are checked against the given IP. @@ -40,8 +69,7 @@ Returns false if the host is not permitted to discover the target device. sub is_discoverable { my $q = shift; - my $device = schema('netdisco')->resultset('Device') - ->search_for_device($q) or return 0; + my $device = get_device($q) or return 0; my $addr = NetAddr::IP::Lite->new($device->ip); my $discover_no = setting('discover_no') || []; @@ -66,32 +94,4 @@ sub is_discoverable { return 1; } -=head2 is_vlan_interface( $port ) - -=cut - -sub is_vlan_interface { - my $port = shift; - - my $is_vlan = (($port->type and - $port->type =~ /^(53|propVirtual|l2vlan|l3ipvlan|135|136|137)$/i) - or ($port->port and $port->port =~ /vlan/i) - or ($port->name and $port->name =~ /vlan/i)) ? 1 : 0; - - return $is_vlan; -} - -=head2 port_has_phone( $port ) - -=cut - -sub port_has_phone { - my $port = shift; - - my $has_phone = ($port->remote_type - and $port->remote_type =~ /ip.phone/i) ? 1 : 0; - - return $has_phone; -} - 1; diff --git a/Netdisco/lib/App/Netdisco/Util/DiscoverAndStore.pm b/Netdisco/lib/App/Netdisco/Util/DiscoverAndStore.pm new file mode 100644 index 00000000..b1aaaf69 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Util/DiscoverAndStore.pm @@ -0,0 +1,693 @@ +package App::Netdisco::Util::DiscoverAndStore; + +use Dancer qw/:syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::DNS 'hostname_from_ip'; +use NetAddr::IP::Lite ':lower'; +use Try::Tiny; + +use base 'Exporter'; +our @EXPORT = (); +our @EXPORT_OK = qw/ + store_device store_interfaces store_wireless + store_vlans store_power store_modules + find_neighbors +/; +our %EXPORT_TAGS = (all => \@EXPORT_OK); + +=head1 NAME + +App::Netdisco::Util::DiscoverAndStore + +=head1 DESCRIPTION + +A set of helper subroutines to support parts of the Netdisco application. + +There are no default exports, however the C<:all> tag will export all +subroutines. + +=head1 EXPORT_OK + +=head2 store_device( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store basic device information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_device { + my ($device, $snmp) = @_; + + my $ip_index = $snmp->ip_index; + my $interfaces = $snmp->interfaces; + my $ip_netmask = $snmp->ip_netmask; + + # build device aliases suitable for DBIC + my @aliases; + foreach my $entry (keys %$ip_index) { + my $ip = NetAddr::IP::Lite->new($entry); + my $addr = $ip->addr; + + next if $addr eq '0.0.0.0'; + next if $ip->within(NetAddr::IP::Lite->new('127.0.0.0/8')); + next if setting('ignore_private_nets') and $ip->is_rfc1918; + + my $iid = $ip_index->{$addr}; + my $port = $interfaces->{$iid}; + my $subnet = $ip_netmask->{$addr} + ? NetAddr::IP::Lite->new($addr, $ip_netmask->{$addr})->network->cidr + : undef; + + debug sprintf ' [%s] store_device - aliased as %s', $device->ip, $addr; + push @aliases, { + alias => $addr, + port => $port, + subnet => $subnet, + dns => hostname_from_ip($addr), + }; + } + + # VTP Management Domain -- assume only one. + my $vtpdomains = $snmp->vtp_d_name; + my $vtpdomain; + if (defined $vtpdomains and scalar values %$vtpdomains) { + $device->vtp_domain( (values %$vtpdomains)[-1] ); + } + + my $hostname = hostname_from_ip($device->ip); + $device->dns($hostname) if length $hostname; + + my @properties = qw/ + snmp_ver snmp_comm + description uptime contact name location + layers ports mac serial model + ps1_type ps2_type ps1_status ps2_status + fan slots + vendor os os_ver + /; + + foreach my $property (@properties) { + $device->$property( $snmp->$property ); + } + + $device->snmp_class( $snmp->class ); + $device->last_discover(\'now()'); + + schema('netdisco')->txn_do(sub { + my $gone = $device->device_ips->delete; + debug sprintf ' [%s] store_device - removed %s aliases', + $device->ip, $gone; + $device->update_or_insert; + $device->device_ips->populate(\@aliases); + debug sprintf ' [%s] store_device - added %d new aliases', + $device->ip, scalar @aliases; + }); +} + +=head2 store_interfaces( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's interface/port information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_interfaces { + my ($device, $snmp) = @_; + + my $interfaces = $snmp->interfaces; + my $i_type = $snmp->i_type; + my $i_ignore = $snmp->i_ignore; + my $i_descr = $snmp->i_description; + my $i_mtu = $snmp->i_mtu; + my $i_speed = $snmp->i_speed; + my $i_mac = $snmp->i_mac; + my $i_up = $snmp->i_up; + my $i_up_admin = $snmp->i_up_admin; + my $i_name = $snmp->i_name; + my $i_duplex = $snmp->i_duplex; + my $i_duplex_admin = $snmp->i_duplex_admin; + my $i_stp_state = $snmp->i_stp_state; + my $i_vlan = $snmp->i_vlan; + my $i_pvid = $snmp->i_pvid; + my $i_lastchange = $snmp->i_lastchange; + + # clear the cached uptime and get a new one + my $dev_uptime = $snmp->load_uptime; + + if (scalar grep {$_ > $dev_uptime} values %$i_lastchange) { + info sprintf ' [%s] interfaces - device uptime has wrapped - correcting', + $device->ip; + $device->uptime( $dev_uptime + 2**32 ); + } + + # build device interfaces suitable for DBIC + my @interfaces; + foreach my $entry (keys %$interfaces) { + my $port = $interfaces->{$entry}; + + if (not length $port) { + debug sprintf ' [%s] interfaces - ignoring %s (no port mapping)', + $device->ip, $port; + next; + } + + if (scalar grep {$port =~ m/$_/} @{setting('ignore_interfaces') || []}) { + debug sprintf + ' [%s] interfaces - ignoring %s (%s) (config:ignore_interfaces)', + $device->ip, $entry, $port; + next; + } + + if (exists $i_ignore->{$entry}) { + debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s)', + $device->ip, $entry, $port, $i_type->{$entry}; + next; + } + + my $lc = $i_lastchange->{$entry}; + if ($device->is_column_changed('uptime') and $lc) { + if ($lc < $dev_uptime) { + # ambiguous: lastchange could be sysUptime before or after wrap + if ($dev_uptime > 30000 and $lc < 30000) { + # uptime wrap more than 5min ago but lastchange within 5min + # assume lastchange was directly after boot -> no action + } + else { + # uptime wrap less than 5min ago or lastchange > 5min ago + # to be on safe side, assume lastchange after counter wrap + debug sprintf + ' [%s] interfaces - correcting LastChange for %s, assuming sysUptime wrap', + $device->ip, $port; + $lc += 2**32; + } + } + } + + push @interfaces, { + port => $port, + descr => $i_descr->{$entry}, + up => $i_up->{$entry}, + up_admin => $i_up_admin->{$entry}, + mac => $i_mac->{$entry}, + speed => $i_speed->{$entry}, + mtu => $i_mtu->{$entry}, + name => $i_name->{$entry}, + duplex => $i_duplex->{$entry}, + duplex_admin => $i_duplex_admin->{$entry}, + stp => $i_stp_state->{$entry}, + type => $i_type->{$entry}, + vlan => $i_vlan->{$entry}, + pvid => $i_pvid->{$entry}, + lastchange => $lc, + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->ports->delete; + debug sprintf ' [%s] interfaces - removed %s interfaces', + $device->ip, $gone; + $device->update_or_insert; + $device->ports->populate(\@interfaces); + debug sprintf ' [%s] interfaces - added %d new interfaces', + $device->ip, scalar @interfaces; + }); +} + +=head2 store_wireless( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's wireless interface information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_wireless { + my ($device, $snmp) = @_; + + my $ssidlist = $snmp->i_ssidlist; + return unless scalar keys %$ssidlist; + + my $interfaces = $snmp->interfaces; + my $ssidbcast = $snmp->i_ssidbcast; + my $ssidmac = $snmp->i_ssidmac; + my $channel = $snmp->i_80211channel; + my $power = $snmp->i_dot11_cur_tx_pwr_mw; + + # build device ssid list suitable for DBIC + my @ssids; + foreach my $entry (keys %$ssidlist) { + $entry =~ s/\.\d+$//; + my $port = $interfaces->{$entry}; + + if (not length $port) { + debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', + $device->ip, $port; + next; + } + + push @ssids, { + port => $port, + ssid => $ssidlist->{$entry}, + broadcast => $ssidbcast->{$entry}, + bssid => $ssidmac->{$entry}, + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->ssids->delete; + debug sprintf ' [%s] wireless - removed %s SSIDs', + $device->ip, $gone; + $device->ssids->populate(\@ssids); + debug sprintf ' [%s] wireless - added %d new SSIDs', + $device->ip, scalar @ssids; + }); + + # build device channel list suitable for DBIC + my @channels; + foreach my $entry (keys %$channel) { + $entry =~ s/\.\d+$//; + my $port = $interfaces->{$entry}; + + if (not length $port) { + debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', + $device->ip, $port; + next; + } + + push @channels, { + port => $port, + channel => $channel->{$entry}, + power => $power->{$entry}, + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->wireless_ports->delete; + debug sprintf ' [%s] wireless - removed %s wireless channels', + $device->ip, $gone; + $device->wireless_ports->populate(\@channels); + debug sprintf ' [%s] wireless - added %d new wireless channels', + $device->ip, scalar @channels; + }); +} + +=head2 store_vlans( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's vlan information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_vlans { + my ($device, $snmp) = @_; + + my $v_name = $snmp->v_name; + my $v_index = $snmp->v_index; + + # build device vlans suitable for DBIC + my %v_seen = (); + my @devicevlans; + foreach my $entry (keys %$v_name) { + my $vlan = $v_index->{$entry}; + ++$v_seen{$vlan}; + + push @devicevlans, { + vlan => $vlan, + description => $v_name->{$entry}, + last_discover => \'now()', + }; + } + + my $i_vlan = $snmp->i_vlan; + my $i_vlan_membership = $snmp->i_vlan_membership; + my $i_vlan_type = $snmp->i_vlan_type; + my $interfaces = $snmp->interfaces; + + # build device port vlans suitable for DBIC + my @portvlans; + foreach my $entry (keys %$i_vlan_membership) { + my $port = $interfaces->{$entry}; + next unless defined $port; + + my $type = $i_vlan_type->{$entry}; + + foreach my $vlan (@{ $i_vlan_membership->{$entry} }) { + my $native = ((defined $i_vlan->{$entry}) and ($vlan eq $i_vlan->{$entry})) ? "t" : "f"; + push @portvlans, { + port => $port, + vlan => $vlan, + native => $native, + vlantype => $type, + last_discover => \'now()', + }; + + next if $v_seen{$vlan}; + + # also add an unnamed vlan to the device + push @devicevlans, { + vlan => $vlan, + description => (sprintf "VLAN %d", $vlan), + last_discover => \'now()', + }; + ++$v_seen{$vlan}; + } + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->vlans->delete; + debug sprintf ' [%s] vlans - removed %s device VLANs', + $device->ip, $gone; + $device->vlans->populate(\@devicevlans); + debug sprintf ' [%s] vlans - added %d new device VLANs', + $device->ip, scalar @devicevlans; + }); + + schema('netdisco')->txn_do(sub { + my $gone = $device->port_vlans->delete; + debug sprintf ' [%s] vlans - removed %s port VLANs', + $device->ip, $gone; + $device->port_vlans->populate(\@portvlans); + debug sprintf ' [%s] vlans - added %d new port VLANs', + $device->ip, scalar @portvlans; + }); +} + +=head2 store_power( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's PoE information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_power { + my ($device, $snmp) = @_; + + my $p_watts = $snmp->peth_power_watts; + my $p_status = $snmp->peth_power_status; + + if (!defined $p_watts) { + debug sprintf ' [%s] power - 0 power modules', $device->ip; + return; + } + + # build device module power info suitable for DBIC + my @devicepower; + foreach my $entry (keys %$p_watts) { + push @devicepower, { + module => $entry, + power => $p_watts->{$entry}, + status => $p_status->{$entry}, + }; + } + + my $interfaces = $snmp->interfaces; + my $p_ifindex = $snmp->peth_port_ifindex; + my $p_admin = $snmp->peth_port_admin; + my $p_pstatus = $snmp->peth_port_status; + my $p_class = $snmp->peth_port_class; + my $p_power = $snmp->peth_port_power; + + # build device port power info suitable for DBIC + my @portpower; + foreach my $entry (keys %$p_ifindex) { + my $port = $interfaces->{ $p_ifindex->{$entry} }; + next unless $port; + + my ($module) = split m/\./, $entry; + + push @portpower, { + port => $port, + module => $module, + admin => $p_admin->{$entry}, + status => $p_pstatus->{$entry}, + class => $p_class->{$entry}, + power => $p_power->{$entry}, + + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->power_modules->delete; + debug sprintf ' [%s] power - removed %s power modules', + $device->ip, $gone; + $device->power_modules->populate(\@devicepower); + debug sprintf ' [%s] power - added %d new power modules', + $device->ip, scalar @devicepower; + }); + + schema('netdisco')->txn_do(sub { + my $gone = $device->powered_ports->delete; + debug sprintf ' [%s] power - removed %s PoE capable ports', + $device->ip, $gone; + $device->powered_ports->populate(\@portpower); + debug sprintf ' [%s] power - added %d new PoE capable ports', + $device->ip, scalar @portpower; + }); +} + +=head2 store_modules( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's module information. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub store_modules { + my ($device, $snmp) = @_; + + my $e_index = $snmp->e_index; + + if (!defined $e_index) { + debug sprintf ' [%s] modules - 0 chassis components', $device->ip; + return; + } + + my $e_descr = $snmp->e_descr; + my $e_type = $snmp->e_type; + my $e_parent = $snmp->e_parent; + my $e_name = $snmp->e_name; + my $e_class = $snmp->e_class; + my $e_pos = $snmp->e_pos; + my $e_hwver = $snmp->e_hwver; + my $e_fwver = $snmp->e_fwver; + my $e_swver = $snmp->e_swver; + my $e_model = $snmp->e_model; + my $e_serial = $snmp->e_serial; + my $e_fru = $snmp->e_fru; + + # build device modules list for DBIC + my @modules; + foreach my $entry (keys %$e_class) { + push @modules, { + index => $e_index->{$entry}, + type => $e_type->{$entry}, + parent => $e_parent->{$entry}, + name => $e_name->{$entry}, + class => $e_class->{$entry}, + pos => $e_pos->{$entry}, + hw_ver => $e_hwver->{$entry}, + fw_ver => $e_fwver->{$entry}, + sw_ver => $e_swver->{$entry}, + model => $e_model->{$entry}, + serial => $e_serial->{$entry}, + fru => $e_fru->{$entry}, + description => $e_descr->{$entry}, + last_discover => \'now()', + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->modules->delete; + debug sprintf ' [%s] modules - removed %s chassis modules', + $device->ip, $gone; + $device->modules->populate(\@modules); + debug sprintf ' [%s] modules - added %d new chassis modules', + $device->ip, scalar @modules; + }); +} + +=head2 find_neighbors( $device, $snmp ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's port neighbors information. + +If any neighbor is unknown to Netdisco, a discover job for it will immediately +be queued (modulo configuration file C setting). + +The Device database object can be a fresh L object which is +not yet stored to the database. + +=cut + +sub find_neighbors { + my ($device, $snmp) = @_; + + my $c_ip = $snmp->c_ip; + unless ($snmp->hasCDP or scalar keys %$c_ip) { + debug sprintf ' [%s] neigh - CDP/LLDP not enabled!', $device->ip; + return; + } + + my $interfaces = $snmp->interfaces; + my $c_if = $snmp->c_if; + my $c_port = $snmp->c_port; + my $c_id = $snmp->c_id; + my $c_platform = $snmp->c_platform; + + foreach my $entry (keys %$c_ip) { + my $port = $interfaces->{ $c_if->{$entry} }; + if (!defined $port) { + debug sprintf ' [%s] neigh - port for IID:%s not resolved, skipping', + $device->ip, $entry; + next; + } + + my $remote_ip = $c_ip->{$entry}; + my $remote_ipad = NetAddr::IP::Lite->new($remote_ip); + my $remote_port = undef; + my $remote_type = $c_platform->{$entry}; + my $remote_id = $c_id->{$entry}; + + next unless length $remote_ip; + + # a bunch of heuristics to search known devices if we don't have a + # useable remote IP... + + if ($remote_ip eq '0.0.0.0' or + $remote_ipad->within(NetAddr::IP::Lite->new('127.0.0.0/8'))) { + + if ($remote_id) { + my $devices = schema('netdisco')->resultset('Device'); + my $neigh = $devices->single({name => $remote_id}); + info sprintf + ' [%s] neigh - bad address %s on port %s, searching for %s instead', + $device->ip, $remote_ip, $port, $remote_id; + + if (!defined $neigh) { + (my $shortid = $remote_id) =~ s/\..*//; + $neigh = $devices->single({name => { -ilike => "${shortid}%" }}); + } + + if ($neigh) { + $remote_ip = $neigh->ip; + info sprintf ' [%s] neigh - found %s with IP %s', + $device->ip, $remote_id, $remote_ip; + } + else { + info sprintf ' [%s] neigh - could not find %s, skipping', + $device->ip, $remote_id; + next; + } + } + else { + info sprintf ' [%s] neigh - skipping unuseable address %s on port %s', + $device->ip, $remote_ip, $port; + next; + } + } + + # hack for devices seeing multiple neighbors on the port + if (ref [] eq ref $remote_ip) { + debug sprintf + ' [%s] neigh - port %s has multiple neighbors, setting remote as self', + $device->ip, $port; + + foreach my $n (@$remote_ip) { + debug sprintf + ' [%s] neigh - adding neighbor %s, type [%s], on %s to discovery queue', + $device->ip, $n, $remote_type, $port; + _enqueue_discover($n, $remote_type); + } + + # set self as remote IP to suppress any further work + $remote_ip = $device->ip; + $remote_port = $port; + } + else { + $remote_port = $c_port->{$entry}; + + if (defined $remote_port) { + # clean weird characters + $remote_port =~ s/[^\d\/\.,()\w:-]+//gi; + } + else { + info sprintf ' [%s] neigh - no remote port found for port %s at %s', + $device->ip, $port, $remote_ip; + } + } + + # XXX too custom? IP Phone detection + if (defined $remote_type and $remote_type =~ m/(mitel.5\d{3})/i) { + $remote_type = 'IP Phone - '. $remote_type + if $remote_type !~ /ip phone/i; + } + + my $portrow = schema('netdisco')->resultset('DevicePort') + ->single({ip => $device->ip, port => $port}); + + if (!defined $portrow) { + info sprintf ' [%s] neigh - local port %s not in database!', + $device->ip, $port; + next; + } + + $portrow->update({ + remote_ip => $remote_ip, + remote_port => $remote_port, + remote_type => $remote_type, + remote_id => $remote_id, + }); + + debug sprintf + ' [%s] neigh - adding neighbor %s, type [%s], on %s to discovery queue', + $device->ip, $remote_ip, $remote_type, $port; + _enqueue_discover($remote_ip, $remote_type); + } +} + +# only enqueue if device is not already discovered, and +# discover_no_type config permits the discovery +sub _enqueue_discover { + my ($ip, $remote_type) = @_; + + my $device = get_device($ip); + return if $device->in_storage; + + my $remote_type_match = setting('discover_no_type'); + if ($remote_type and $remote_type_match + and $remote_type =~ m/$remote_type_match/) { + debug sprintf ' queue - %s, type [%s] excluded by discover_no_type', + $ip, $remote_type; + return; + } + + try { + # could fail if queued job already exists + schema('netdisco')->resultset('Admin')->create({ + device => $ip, + action => 'discover', + status => 'queued', + }); + }; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Util/Permissions.pm b/Netdisco/lib/App/Netdisco/Util/Permissions.pm deleted file mode 100644 index 96b6a9a7..00000000 --- a/Netdisco/lib/App/Netdisco/Util/Permissions.pm +++ /dev/null @@ -1,74 +0,0 @@ -package App::Netdisco::Util::Permissions; - -use Dancer qw/:syntax :script/; -use Dancer::Plugin::DBIC 'schema'; - -use App::Netdisco::Util::DeviceProperties ':all'; - -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ - vlan_reconfig_check port_reconfig_check -/; -our %EXPORT_TAGS = ( - all => [qw/ - vlan_reconfig_check port_reconfig_check - /], -); - -=head1 App::Netdisco::Util::Permissions - -A set of helper subroutines to support parts of the Netdisco application. - -There are no default exports, however the C<:all> tag will export all -subroutines. - -=head2 vlan_reconfig_check( $port ) - -=cut - -sub vlan_reconfig_check { - my $port = shift; - my $ip = $port->ip; - my $name = $port->port; - - my $is_vlan = is_vlan_interface($port); - - # vlan (routed) interface check - return "forbidden: [$name] is a vlan interface on [$ip]" - if $is_vlan; - - return "forbidden: not permitted to change native vlan" - if not setting('vlanctl'); - - return; -} - -=head2 port_reconfig_check( $port ) - -=cut - -sub port_reconfig_check { - my $port = shift; - my $ip = $port->ip; - my $name = $port->port; - - my $has_phone = port_has_phone($port); - my $is_vlan = is_vlan_interface($port); - - # uplink check - return "forbidden: port [$name] on [$ip] is an uplink" - if $port->remote_type and not $has_phone and not setting('allow_uplinks'); - - # phone check - return "forbidden: port [$name] on [$ip] is a phone" - if $has_phone and setting('portctl_nophones'); - - # vlan (routed) interface check - return "forbidden: [$name] is a vlan interface on [$ip]" - if $is_vlan and not setting('portctl_vlans'); - - return; -} - -1; diff --git a/Netdisco/lib/App/Netdisco/Util/Port.pm b/Netdisco/lib/App/Netdisco/Util/Port.pm new file mode 100644 index 00000000..81f6548f --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Util/Port.pm @@ -0,0 +1,224 @@ +package App::Netdisco::Util::Port; + +use Dancer qw/:syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::Device 'get_device'; + +use base 'Exporter'; +our @EXPORT = (); +our @EXPORT_OK = qw/ + vlan_reconfig_check port_reconfig_check + get_port get_iid get_powerid + is_vlan_interface port_has_phone +/; +our %EXPORT_TAGS = (all => \@EXPORT_OK); + +=head1 NAME + +App::Netdisco::Util::Port + +=head1 DESCRIPTION + +A set of helper subroutines to support parts of the Netdisco application. + +There are no default exports, however the C<:all> tag will export all +subroutines. + +=head1 EXPORT_OK + +=head2 vlan_reconfig_check( $port ) + +=over 4 + +=item * + +Sanity check that C<$port> is not a vlan subinterface. + +=item * + +Permission check that C is true in Netdisco config. + +=back + +Will return nothing if these checks pass OK. + +=cut + +sub vlan_reconfig_check { + my $port = shift; + my $ip = $port->ip; + my $name = $port->port; + + my $is_vlan = is_vlan_interface($port); + + # vlan (routed) interface check + return "forbidden: [$name] is a vlan interface on [$ip]" + if $is_vlan; + + return "forbidden: not permitted to change native vlan" + if not setting('vlanctl'); + + return; +} + +=head2 port_reconfig_check( $port ) + +=over 4 + +=item * + +Permission check that C is true in Netdisco config, if C<$port> +is an uplink. + +=item * + +Permission check that C is not true in Netdisco config, if +C<$port> has a phone connected. + +=item * + +Permission check that C is true if C<$port> is a vlan +subinterface. + +=back + +Will return nothing if these checks pass OK. + +=cut + +sub port_reconfig_check { + my $port = shift; + my $ip = $port->ip; + my $name = $port->port; + + my $has_phone = port_has_phone($port); + my $is_vlan = is_vlan_interface($port); + + # uplink check + return "forbidden: port [$name] on [$ip] is an uplink" + if $port->remote_type and not $has_phone and not setting('allow_uplinks'); + + # phone check + return "forbidden: port [$name] on [$ip] is a phone" + if $has_phone and setting('portctl_nophones'); + + # vlan (routed) interface check + return "forbidden: [$name] is a vlan interface on [$ip]" + if $is_vlan and not setting('portctl_vlans'); + + return; +} + +=head2 get_port( $device, $portname ) + +Given a device IP address and a port name, returns a L +object for the Port on the Device in the Netdisco database. + +The device IP can also be passed as a Device C object. + +Returns C if the device or port are not known to Netdisco. + +=cut + +sub get_port { + my ($device, $portname) = @_; + + # accept either ip or dbic object + $device = get_device($device); + + my $port = schema('netdisco')->resultset('DevicePort') + ->find({ip => $device->ip, port => $portname}); + + return $port; +} + +=head2 get_iid( $info, $port ) + +Given an L instance for a device, and the name of a port, returns +the current interface table index for that port. This can be used in further +SNMP requests on attributes of the port. + +Returns C if there is no such port name on the device. + +=cut + +sub get_iid { + my ($info, $port) = @_; + + # accept either port name or dbic object + $port = $port->port if ref $port; + + my $interfaces = $info->interfaces; + my %rev_if = reverse %$interfaces; + my $iid = $rev_if{$port}; + + return $iid; +} + +=head2 get_powerid( $info, $port ) + +Given an L instance for a device, and the name of a port, returns +the current PoE table index for the port. This can be used in further SNMP +requests on PoE attributes of the port. + +Returns C if there is no such port name on the device. + +=cut + +sub get_powerid { + my ($info, $port) = @_; + + # accept either port name or dbic object + $port = $port->port if ref $port; + + my $iid = get_iid($info, $port) + or return undef; + + my $p_interfaces = $info->peth_port_ifindex; + my %rev_p_if = reverse %$p_interfaces; + my $powerid = $rev_p_if{$iid}; + + return $powerid; +} + +=head2 is_vlan_interface( $port ) + +Returns true if the C<$port> L object represents a vlan +subinterface. + +This uses simple checks on the port I and I, and therefore might +sometimes returns a false-negative result. + +=cut + +sub is_vlan_interface { + my $port = shift; + + my $is_vlan = (($port->type and + $port->type =~ /^(53|propVirtual|l2vlan|l3ipvlan|135|136|137)$/i) + or ($port->port and $port->port =~ /vlan/i) + or ($port->name and $port->name =~ /vlan/i)) ? 1 : 0; + + return $is_vlan; +} + +=head2 port_has_phone( $port ) + +Returns true if the C<$port> L object has a phone connected. + +This uses a simple check on the I of the remote connected device, and +therefore might sometimes return a false-negative result. + +=cut + +sub port_has_phone { + my $port = shift; + + my $has_phone = ($port->remote_type + and $port->remote_type =~ /ip.phone/i) ? 1 : 0; + + return $has_phone; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Util/SNMP.pm b/Netdisco/lib/App/Netdisco/Util/SNMP.pm new file mode 100644 index 00000000..d233ac2c --- /dev/null +++ b/Netdisco/lib/App/Netdisco/Util/SNMP.pm @@ -0,0 +1,105 @@ +package App::Netdisco::Util::SNMP; + +use Dancer qw/:syntax :script/; +use App::Netdisco::Util::Device 'get_device'; + +use SNMP::Info; +use Try::Tiny; +use Path::Class 'dir'; + +use base 'Exporter'; +our @EXPORT = (); +our @EXPORT_OK = qw/ + snmp_connect snmp_connect_rw +/; +our %EXPORT_TAGS = (all => \@EXPORT_OK); + +=head1 NAME + +App::Netdisco::Util::SNMP + +=head1 DESCRIPTION + +A set of helper subroutines to support parts of the Netdisco application. + +There are no default exports, however the C<:all> tag will export all +subroutines. + +=head1 EXPORT_OK + +=head2 snmp_connect( $ip ) + +Given an IP address, returns an L instance configured for and +connected to that device. The IP can be any on the device, and the management +interface will be connected to. + +If the device is known to Netdisco and there is a cached SNMP community +string, this will be tried first, and then other community string(s) from the +application configuration will be tried. + +Returns C if the connection fails. + +=cut + +sub snmp_connect { _snmp_connect_generic(@_, 'community') } + +=head2 snmp_connect_rw( $ip ) + +Same as C but uses the read-write community string(s) from the +application configuration file. + +Returns C if the connection fails. + +=cut + +sub snmp_connect_rw { _snmp_connect_generic(@_, 'community_rw') } + +sub _snmp_connect_generic { + my $ip = shift; + + # get device details from db + my $device = get_device($ip); + + # get the community string(s) + my $comm_type = pop; + my @communities = @{ setting($comm_type) || []}; + unshift @communities, $device->snmp_comm + if length $device->snmp_comm + and length $comm_type and $comm_type eq 'community'; + + # TODO: only supporing v2c at the moment + my %snmp_args = ( + DestHost => $device->ip, + Version => ($device->snmp_ver || setting('snmpver') || 2), + Retries => (setting('snmpretries') || 2), + Timeout => (setting('snmptimeout') || 1000000), + MibDirs => [ _build_mibdirs() ], + AutoSpecify => 1, + IgnoreNetSNMPConf => 1, + Debug => ($ENV{INFO_TRACE} || 0), + ); + + my $info = undef; + my $last_comm = 0; + COMMUNITY: foreach my $c (@communities) { + next unless defined $c and length $c; + try { + $info = SNMP::Info->new(%snmp_args, Community => $c); + ++$last_comm if ( + $info + and (not defined $info->error) + and length $info->uptime + ); + }; + last COMMUNITY if $last_comm; + } + + return $info; +} + +sub _build_mibdirs { + return map { dir(setting('mibhome'), $_) } + @{ setting('mibdirs') || [] }; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/Util/Web.pm b/Netdisco/lib/App/Netdisco/Util/Web.pm index 290065a9..b02aa861 100644 --- a/Netdisco/lib/App/Netdisco/Util/Web.pm +++ b/Netdisco/lib/App/Netdisco/Util/Web.pm @@ -5,19 +5,21 @@ our @EXPORT = (); our @EXPORT_OK = qw/ sort_port /; -our %EXPORT_TAGS = ( - all => [qw/ - sort_port - /], -); +our %EXPORT_TAGS = (all => \@EXPORT_OK); -=head1 App::Netdisco::Util::Web +=head1 NAME + +App::Netdisco::Util::Web + +=head1 DESCRIPTION A set of helper subroutines to support parts of the Netdisco application. There are no default exports, however the C<:all> tag will export all subroutines. +=head1 EXPORT_OK + =head2 sort_port( $a, $b ) Sort port names of various types used by device vendors. Interface is as diff --git a/Netdisco/share/config.yml b/Netdisco/share/config.yml index aa797a8a..cc13a5ad 100644 --- a/Netdisco/share/config.yml +++ b/Netdisco/share/config.yml @@ -18,6 +18,9 @@ charset: "UTF-8" # web sessions stored on disk session: "YAML" +# HTTP port to listen on +port: 5000 + # logging format logger_format: '[%P] %L @%D> %m' @@ -70,8 +73,32 @@ portctl_vlans: 1 # sleep time between polls of the database job queue daemon_sleep_time: 5 -# how many daemon processes -# NB one worker will always be a Queue Manager -daemon_pollers: 0 +# how many daemon processes for this node daemon_interactives: 2 +daemon_pollers: 2 +# what housekeeping tasks should this node *schedule* +# (it only does them if daemon_pollers is non-zero) +#housekeeping: +# discoverall: +# device: '192.0.2.0' +# when: +# wday: 'wed' +# hour: 14 +# backup: +# when: +# hour: 1 +# refresh: +# when: '0 9 * * *' +# arpwalk: +# when: +# min: 30 +# macwalk: +# when: +# hour: '*/2' +# nbtwalk: +# when: +# hour: '8,13,21' +# saveconfigs: +# param: 61 +# when: '0 * * * *' diff --git a/TODO b/TODO index a8b17ddf..68d09f01 100644 --- a/TODO +++ b/TODO @@ -7,15 +7,15 @@ FRONTEND * UI for topo DB table editing - drop topo file support and use DB only -* (jeneric) device module tab * Port/Name/VLAN box should be green when filled +* Choice of MAC address formats +* Empty inventory should trigger request to discover + +* (jeneric) device module tab DAEMON ====== -* poller daemon (scheduling automatically?) - - plugins for poller - CORE ====