diff --git a/Build.PL b/Build.PL index 9466f1d2..bf358acc 100644 --- a/Build.PL +++ b/Build.PL @@ -17,6 +17,7 @@ Module::Build->new( }, requires => { 'aliased' => '0', + 'autovivification' => '0', 'namespace::clean' => '0.24', 'version' => '0.9902', 'Algorithm::Cron' => '0.07', @@ -46,7 +47,7 @@ Module::Build->new( 'JSON' => '2.90', 'JSON::XS' => '3.01', 'List::Util' => '1.49', - 'List::MoreUtils' => '0.33', + 'List::MoreUtils' => '0.428', 'MIME::Base64' => '3.13', 'Module::Load' => '0.32', 'Moo' => '1.001000', @@ -72,7 +73,7 @@ Module::Build->new( 'Starman' => '0.4008', 'Storable' => '0', 'Sys::SigAction' => '0', - 'SNMP::Info' => '3.64', + 'SNMP::Info' => '3.65', 'SQL::Abstract' => '1.85', 'SQL::Translator' => '0.11024', 'Template' => '2.24', diff --git a/Changes b/Changes index 358c0e9e..4f29f214 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,76 @@ +2.040007 - 2019-03-06 + + [BUG FIXES] + + * #521-redux Search Node Date Range not working (ollyg) + +2.040006 - 2019-03-04 + + [BUG FIXES] + + * #527 update List::MoreUtils version requirement + +2.040005 - 2019-03-04 + + [BUG FIXES] + + * #526 fix discover syntax bug + +2.040004 - 2019-03-03 + + [NEW FEATURES] + + * #510 store ifindex in Device Port Properties table (rc9000) + * new discover_waps and discover_phones boolean settings (ollyg) + + [ENHANCEMENTS] + + * #428 Port-Channels now showing in netmap (ollyg) + * #490 use new LLDP capability checks for ports having phones (ollyg) + * #494 update Cisco ASA ssh collector (stromsoe) + + [BUG FIXES] + + * #492 Port Control incorrectly uses VLAN config check (inphobia) + * #493 HTML tag fix (inphobia) + * #498 Map with VLAN filter omits unconnected devices (ollyg) + * #499 netdisco-do renumber reports wrong ip (inphobia) + * #500 no more duplicate entries in vlan filter (ollyg) + * #505 renumbering device missed a few tables (ollyg) + * #512 fix regression in phone/wap discovery exclusion (ollyg) + * #514 ipinventory report returns consistent data (inphobia) + * #520 make sure aggports have a master<->slave (ollyg) + * #521 Search Node Date Range not working (ollyg) + * #522 TypeAhead.pm can reference empty data (inphobia) + * fix bug showing no nodes when only one matches in netmap (ollyg) + +2.040003 - 2019-01-18 + + [NEW FEATURES] + + * #485 new "VLANs" device tab showing a VLAN report (inphobia) + + [ENHANCEMENTS] + + * #408, #417 & 477 makerancidconf improvements re-added (earendilfr, inphobia) + * #420 IP Inventory Node column renamed to IP Address (ollyg) + * #420 sidebar defaults for IP Inventory report can be overridden (ollyg) + * #424 column name is "Connected Nodes & Devices" when both are shown (ollyg) + * #436 make neighbor matching less strict in netmap (linwood-f) + * #482 operating system is now a link in device details (inphobia) + * #486 allow snmp::info base class in netdisco-do (inphobia) + + [BUG FIXES] + + * #457 make sorting work for adresses when interface was undefined (inphobia) + * #471-redux ospf discovery will now keep on working (ollyg) + * #474 better explain public key auth with netdisco-sshcollector (inphobia) + * #475, #479 all discover plugins should now respect ignore_* for interfaces (ollyg) + * #476 debug log when deleting rows from related tables (ollyg) + * Fix to catch when txrate on wifi is scalar and not a list + * Remove 'use vars' which is deprecated in Perl + * Various documentation improvements (inphobia) + 2.040002 - 2018-12-30 [BUG FIXES] diff --git a/MANIFEST b/MANIFEST index 6b1c8640..32b20851 100644 --- a/MANIFEST +++ b/MANIFEST @@ -151,6 +151,7 @@ lib/App/Netdisco/Web/Plugin/Device/Details.pm lib/App/Netdisco/Web/Plugin/Device/Modules.pm lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm lib/App/Netdisco/Web/Plugin/Device/Ports.pm +lib/App/Netdisco/Web/Plugin/Device/Vlans.pm lib/App/Netdisco/Web/Plugin/Inventory.pm lib/App/Netdisco/Web/Plugin/Report/ApChannelDist.pm lib/App/Netdisco/Web/Plugin/Report/ApClients.pm @@ -363,6 +364,7 @@ share/schema_versions/App-Netdisco-DB-5-6-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-50-51-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-51-52-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-52-53-PostgreSQL.sql +share/schema_versions/App-Netdisco-DB-53-54-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-6-7-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-7-8-PostgreSQL.sql share/schema_versions/App-Netdisco-DB-8-9-PostgreSQL.sql @@ -391,6 +393,8 @@ share/views/ajax/device/modules.tt share/views/ajax/device/netmap.tt share/views/ajax/device/ports.tt share/views/ajax/device/ports_csv.tt +share/views/ajax/device/vlans.tt +share/views/ajax/device/vlans_csv.tt share/views/ajax/report/apchanneldist.tt share/views/ajax/report/apchanneldist_csv.tt share/views/ajax/report/apclients.tt diff --git a/META.json b/META.json index e58fa24c..c19aca81 100644 --- a/META.json +++ b/META.json @@ -60,7 +60,7 @@ "IO::Socket::SSL" : "2.048", "JSON" : "2.90", "JSON::XS" : "3.01", - "List::MoreUtils" : "0.33", + "List::MoreUtils" : "0.428", "List::Util" : "1.49", "MCE" : "1.703", "MIME::Base64" : "3.13", @@ -81,7 +81,7 @@ "Plack::Middleware::ReverseProxy" : "0.15", "Pod::Usage" : "0", "Role::Tiny" : "1.002005", - "SNMP::Info" : "3.64", + "SNMP::Info" : "3.65", "SQL::Abstract" : "1.85", "SQL::Translator" : "0.11024", "Scope::Guard" : "0", @@ -102,6 +102,7 @@ "YAML" : "0.84", "YAML::XS" : "0.41", "aliased" : "0", + "autovivification" : "0", "namespace::clean" : "0.24", "version" : "0.9902" } @@ -117,7 +118,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.040002" + "version" : "2.040007" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -139,7 +140,7 @@ }, "App::Netdisco::DB" : { "file" : "lib/App/Netdisco/DB.pm", - "version" : "53" + "version" : "54" }, "App::Netdisco::DB::ExplicitLocking" : { "file" : "lib/App/Netdisco/DB/ExplicitLocking.pm" @@ -531,6 +532,9 @@ "App::Netdisco::Web::Plugin::Device::Ports" : { "file" : "lib/App/Netdisco/Web/Plugin/Device/Ports.pm" }, + "App::Netdisco::Web::Plugin::Device::Vlans" : { + "file" : "lib/App/Netdisco/Web/Plugin/Device/Vlans.pm" + }, "App::Netdisco::Web::Plugin::Inventory" : { "file" : "lib/App/Netdisco/Web/Plugin/Inventory.pm" }, @@ -796,6 +800,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.040002", + "version" : "2.040007", "x_serialization_backend" : "JSON::PP version 2.97001" } diff --git a/META.yml b/META.yml index 190948b6..f7cb0c94 100644 --- a/META.yml +++ b/META.yml @@ -22,7 +22,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.040002' + version: '2.040007' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -37,7 +37,7 @@ provides: file: lib/App/Netdisco/Configuration.pm App::Netdisco::DB: file: lib/App/Netdisco/DB.pm - version: '53' + version: '54' App::Netdisco::DB::ExplicitLocking: file: lib/App/Netdisco/DB/ExplicitLocking.pm App::Netdisco::DB::Result::Admin: @@ -298,6 +298,8 @@ provides: file: lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm App::Netdisco::Web::Plugin::Device::Ports: file: lib/App/Netdisco/Web/Plugin/Device/Ports.pm + App::Netdisco::Web::Plugin::Device::Vlans: + file: lib/App/Netdisco/Web/Plugin/Device/Vlans.pm App::Netdisco::Web::Plugin::Inventory: file: lib/App/Netdisco/Web/Plugin/Inventory.pm App::Netdisco::Web::Plugin::Report::ApChannelDist: @@ -495,7 +497,7 @@ requires: IO::Socket::SSL: '2.048' JSON: '2.90' JSON::XS: '3.01' - List::MoreUtils: '0.33' + List::MoreUtils: '0.428' List::Util: '1.49' MCE: '1.703' MIME::Base64: '3.13' @@ -516,7 +518,7 @@ requires: Plack::Middleware::ReverseProxy: '0.15' Pod::Usage: '0' Role::Tiny: '1.002005' - SNMP::Info: '3.64' + SNMP::Info: '3.65' SQL::Abstract: '1.85' SQL::Translator: '0.11024' Scope::Guard: '0' @@ -537,6 +539,7 @@ requires: YAML: '0.84' YAML::XS: '0.41' aliased: '0' + autovivification: '0' namespace::clean: '0.24' version: '0.9902' resources: @@ -546,5 +549,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: '2.040002' +version: '2.040007' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/bin/netdisco-do b/bin/netdisco-do index a720361b..9396f61f 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -301,14 +301,30 @@ the leaf (such as C or C). If you wish to test with a device class other than that discovered, prefix the leaf with the class short name, for example "C" or -"C". +"C". Using "C<::>" as the start of the prefix will test +against the base "C" class. + +As well, SNMP object names can be used as an argument for "C<-e>", so you can +use C for example, which will use the netdisco-mibs files for +translations. + +All "C<-e>" parameters are case sensitive. ~/bin/netdisco-do show -d 192.0.2.1 -e interfaces ~/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces + ~/bin/netdisco-do show -d 192.0.2.1 -e ::interfaces + ~/bin/netdisco-do show -d 192.0.2.1 -e ifName -A parameter may be passed to the C method in the C<-p> parameter: +A parameter may be passed to the C method or SNMP object in the +"C<-p>" parameter: ~/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3 + ~/bin/netdisco-do show -d 192.0.2.1 -e ifName -p 2 + +The "C<-e>" parameter C will show the used configuration for the +specified device. + + ~/bin/netdisco-do show -d 192.0.2.1 -e specify =head2 psql diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 04b66052..29c0e154 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -185,7 +185,7 @@ full SNMP support # install dependencies: ~/bin/localenv cpanm --notest Net::OpenSSH Expect - + # run manually, or add to cron: ~/bin/netdisco-sshcollector [-DQO] [-w ] @@ -209,10 +209,10 @@ Currently, ARP tables can be retrieved from the following device classes: =item * L - Cisco IOS -=item * L - Cisco NXOS - =item * L - Cisco IOS XR +=item * L - Cisco NXOS + =item * L - F5 Networks BigIP =item * L - FreeBSD @@ -227,8 +227,8 @@ The collected arp entries are then directly stored in the netdisco database. =head1 CONFIGURATION -The following should go into your Netdisco 2 configuration file, "C<< -~/environments/deployment.yml >>" +The following should go into your Netdisco configuration file, +F<~/environments/deployment.yml>. =over 4 @@ -236,7 +236,7 @@ The following should go into your Netdisco 2 configuration file, "C<< Data is collected from the machines specified in this setting. The format is a list of dictionaries. The keys C, C, C, and C -are required. Optionally the C key can be used instead of the +are required. Optionally the C key can be used instead of the C. For example: sshcollector: @@ -246,14 +246,16 @@ C. For example: platform: IOS - hostname: 'core-router.example.com' user: oliver - password: letmein + password: platform: IOS Platform is the final part of the classname to be instantiated to query the host, e.g. platform B will be queried using C. -If the password is "-", public key authentication will be attempted. +If the password is blank, public key authentication will be attempted with the +default key for the netdisco user. Password protected keys are currently not +supported. =back @@ -294,7 +296,7 @@ B and B datatypes in PostgreSQL can handle. =item C<-D> -Netdisco debug log level +Netdisco debug log level. =item C<-Q> @@ -302,7 +304,7 @@ L trace enabled. =item C<-O> -L trace enabled +L trace enabled. =item C<-w> @@ -310,9 +312,8 @@ Set maximum parallel workers for L. The default is B. =item C<-d device> -Only run for a single device. Takes an IP or hostname, must exactly match the value -in the config file. - +Only run for a single device. Takes an IP or hostname, must exactly match the +value in the config file. =back @@ -326,6 +327,8 @@ in the config file. =item L +=item L + =back - + =cut diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 7904c0b0..cecfe6ae 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.040002'; +our $VERSION = '2.040007'; use App::Netdisco::Configuration; =head1 NAME @@ -312,7 +312,7 @@ of Waikato, Hamilton NZ), Dusty Hall (Auburn U), Jon Monroe (center pointe), Alexander Barthel, Bill Anderson, Alexander Hartmaier (t-systems.at), Justin Hunter (Arizona State U), Jethro Binks (U of Strathclyde, Glasgow), Jordi Guijarro (UAB.es), Sam Stickland (spacething.org), Stefan Radman (CTBTO.org), -Clint Wise, Max Kosmach, and Bernhard Augenstein. +Clint Wise, Max Kosmach, Bernhard Augenstein and Nick Nauwelaerts (aquafin.be). We probably forgot some names - sorry about that :-(. diff --git a/lib/App/Netdisco/DB/Result/Device.pm b/lib/App/Netdisco/DB/Result/Device.pm index a6e06545..412ecdaa 100644 --- a/lib/App/Netdisco/DB/Result/Device.pm +++ b/lib/App/Netdisco/DB/Result/Device.pm @@ -262,20 +262,27 @@ sub renumber { foreach my $set (qw/ DeviceIp DeviceModule - DevicePower - DeviceVlan DevicePort DevicePortLog DevicePortPower + DevicePortProperties DevicePortSsid DevicePortVlan DevicePortWireless + DevicePower + DeviceVlan /) { $schema->resultset($set) ->search({ip => $old_ip}) ->update({ip => $new_ip}); } + $schema->resultset('DeviceSkip') + ->search({device => $new_ip})->delete; + $schema->resultset('DeviceSkip') + ->search({device => $old_ip}) + ->update({device => $new_ip}); + $schema->resultset('DevicePort') ->search({remote_ip => $old_ip}) ->update({remote_ip => $new_ip}); diff --git a/lib/App/Netdisco/DB/Result/DevicePortProperties.pm b/lib/App/Netdisco/DB/Result/DevicePortProperties.pm index 81ea937b..b25fdaaf 100644 --- a/lib/App/Netdisco/DB/Result/DevicePortProperties.pm +++ b/lib/App/Netdisco/DB/Result/DevicePortProperties.pm @@ -29,6 +29,8 @@ __PACKAGE__->add_columns( { data_type => "bigint", is_nullable => 1 }, "faststart", { data_type => "boolean", is_nullable => 1 }, + "ifindex", + { data_type => "bigint", is_nullable => 1 }, ); __PACKAGE__->set_primary_key("port", "ip"); diff --git a/lib/App/Netdisco/DB/Result/Virtual/DeviceLinks.pm b/lib/App/Netdisco/DB/Result/Virtual/DeviceLinks.pm index f59c315a..9d2339ca 100644 --- a/lib/App/Netdisco/DB/Result/Virtual/DeviceLinks.pm +++ b/lib/App/Netdisco/DB/Result/Virtual/DeviceLinks.pm @@ -15,32 +15,61 @@ __PACKAGE__->table_class('DBIx::Class::ResultSource::View'); __PACKAGE__->table('device_links'); __PACKAGE__->result_source_instance->is_virtual(1); __PACKAGE__->result_source_instance->view_definition(<result_source_instance->view_definition(<<'ENDSQL'); d.ip, d.name, d.dns, p.port, p.name AS port_description, p.remote_ip, p.remote_id, p.remote_type, p.remote_port, + dpp.remote_is_wap, dpp.remote_is_phone, l.log AS comment, a.log, a.finished @@ -23,6 +24,7 @@ __PACKAGE__->result_source_instance->view_definition(<<'ENDSQL'); INNER JOIN device d USING (ip) LEFT OUTER JOIN device_skip ds ON ('discover' = ANY(ds.actionset) AND p.remote_ip = ds.device) + LEFT OUTER JOIN device_port_properties dpp USING (ip, port) LEFT OUTER JOIN device_port_log l USING (ip, port) LEFT OUTER JOIN admin a ON (p.remote_ip = a.device AND a.action = 'discover') @@ -58,6 +60,10 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 1 }, "remote_id", { data_type => "text", is_nullable => 1 }, + "remote_is_wap", + { data_type => "boolean", is_nullable => 1 }, + "remote_is_phone", + { data_type => "boolean", is_nullable => 1 }, "comment", { data_type => "text", is_nullable => 1 }, "log", diff --git a/lib/App/Netdisco/DB/ResultSet/Device.pm b/lib/App/Netdisco/DB/ResultSet/Device.pm index 1d755de3..271eeda6 100644 --- a/lib/App/Netdisco/DB/ResultSet/Device.pm +++ b/lib/App/Netdisco/DB/ResultSet/Device.pm @@ -3,7 +3,10 @@ use base 'App::Netdisco::DB::ResultSet'; use strict; use warnings; + +use Try::Tiny; use NetAddr::IP::Lite ':lower'; +require Dancer::Logger; =head1 ADDITIONAL METHODS @@ -44,16 +47,16 @@ sub with_times { ->search({}, { '+columns' => { - uptime_age => \("replace(age(timestamp 'epoch' + uptime / 100 * interval '1 second', " + uptime_age => \("replace(age(timestamp 'epoch' + me.uptime / 100 * interval '1 second', " ."timestamp '1970-01-01 00:00:00-00')::text, 'mon', 'month')"), first_seen_stamp => \"to_char(me.creation, 'YYYY-MM-DD HH24:MI')", - last_discover_stamp => \"to_char(last_discover, 'YYYY-MM-DD HH24:MI')", - last_macsuck_stamp => \"to_char(last_macsuck, 'YYYY-MM-DD HH24:MI')", - last_arpnip_stamp => \"to_char(last_arpnip, 'YYYY-MM-DD HH24:MI')", + last_discover_stamp => \"to_char(me.last_discover, 'YYYY-MM-DD HH24:MI')", + last_macsuck_stamp => \"to_char(me.last_macsuck, 'YYYY-MM-DD HH24:MI')", + last_arpnip_stamp => \"to_char(me.last_arpnip, 'YYYY-MM-DD HH24:MI')", since_first_seen => \"extract(epoch from (age(now(), me.creation)))", - since_last_discover => \"extract(epoch from (age(now(), last_discover)))", - since_last_macsuck => \"extract(epoch from (age(now(), last_macsuck)))", - since_last_arpnip => \"extract(epoch from (age(now(), last_arpnip)))", + since_last_discover => \"extract(epoch from (age(now(), me.last_discover)))", + since_last_macsuck => \"extract(epoch from (age(now(), me.last_macsuck)))", + since_last_arpnip => \"extract(epoch from (age(now(), me.last_arpnip)))", }, }); } @@ -591,12 +594,22 @@ handle the removal or archiving of nodes. =cut +sub _plural { (shift || 0) == 1 ? 'entry' : 'entries' }; + sub delete { my $self = shift; my $schema = $self->result_source->schema; my $devices = $self->search(undef, { columns => 'ip' }); + my $ip = undef; + { + no autovivification; + try { $ip ||= $devices->{attrs}->{where}->{ip} }; + try { $ip ||= $devices->{attrs}->{where}->{'me.ip'} }; + } + $ip ||= 'netdisco'; + foreach my $set (qw/ DeviceIp DeviceVlan @@ -604,9 +617,12 @@ sub delete { DeviceModule Community /) { - $schema->resultset($set)->search( + my $gone = $schema->resultset($set)->search( { ip => { '-in' => $devices->as_query } }, )->delete; + + Dancer::Logger::debug sprintf ' [%s] db/device - removed %d %s from %s', + $ip, $gone, _plural($gone), $set if defined Dancer::Logger::logger(); } foreach my $set (qw/ @@ -618,13 +634,16 @@ sub delete { )->delete; } - $schema->resultset('Topology')->search({ + my $gone = $schema->resultset('Topology')->search({ -or => [ { dev1 => { '-in' => $devices->as_query } }, { dev2 => { '-in' => $devices->as_query } }, ], })->delete; + Dancer::Logger::debug sprintf ' [%s] db/device - removed %d manual topology %s', + $ip, $gone, _plural($gone) if defined Dancer::Logger::logger(); + $schema->resultset('DevicePort')->search( { ip => { '-in' => $devices->as_query } }, )->delete(@_); diff --git a/lib/App/Netdisco/DB/ResultSet/DevicePort.pm b/lib/App/Netdisco/DB/ResultSet/DevicePort.pm index cf0c978c..3ae4f80f 100644 --- a/lib/App/Netdisco/DB/ResultSet/DevicePort.pm +++ b/lib/App/Netdisco/DB/ResultSet/DevicePort.pm @@ -4,6 +4,9 @@ use base 'App::Netdisco::DB::ResultSet'; use strict; use warnings; +use Try::Tiny; +require Dancer::Logger; + __PACKAGE__->load_components(qw/ +App::Netdisco::DB::ExplicitLocking /); @@ -222,12 +225,22 @@ handle the removal or archiving of nodes. =cut +sub _plural { (shift || 0) == 1 ? 'entry' : 'entries' }; + sub delete { my $self = shift; my $schema = $self->result_source->schema; my $ports = $self->search(undef, { columns => 'ip' }); + my $ip = undef; + { + no autovivification; + try { $ip ||= ${ $ports->{attrs}->{where}->{ip}->{'-in'} }->[1]->[1] }; + try { $ip ||= $ports->{attrs}->{where}->{'me.ip'} }; + } + $ip ||= 'netdisco'; + foreach my $set (qw/ DevicePortPower DevicePortProperties @@ -235,9 +248,12 @@ sub delete { DevicePortWireless DevicePortSsid /) { - $schema->resultset($set)->search( + my $gone = $schema->resultset($set)->search( { ip => { '-in' => $ports->as_query }}, )->delete; + + Dancer::Logger::debug sprintf ' [%s] db/ports - removed %d port %s from %s', + $ip, $gone, _plural($gone), $set if defined Dancer::Logger::logger(); } $schema->resultset('Node')->search( diff --git a/lib/App/Netdisco/SSHCollector/Platform/ASA.pm b/lib/App/Netdisco/SSHCollector/Platform/ASA.pm index d28373bf..c0a4e540 100644 --- a/lib/App/Netdisco/SSHCollector/Platform/ASA.pm +++ b/lib/App/Netdisco/SSHCollector/Platform/ASA.pm @@ -75,21 +75,21 @@ sub arpnip { $expect->send( $args->{enable_password} ."\n" ); } - $prompt = qr/#/; + $prompt = qr/#\s*$/; ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt); + $expect->send("terminal pager 2147483647\n"); + ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); + $expect->send("show names\n"); ($pos, $error, $match, $before, $after) = $expect->expect(60, -re, $prompt); my @names = split(m/\n/, $before); - $expect->send("terminal pager 2147483647\n"); - ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); - $expect->send("show arp\n"); ($pos, $error, $match, $before, $after) = $expect->expect(60, -re, $prompt); + my @lines = split(m/\n/, $before); my @arpentries = (); - my @lines = split(m/\n/, $before); # ifname 192.0.2.1 0011.2233.4455 123 my $linereg = qr/[A-z0-9\-\.]+\s([A-z0-9\-\.]+)\s @@ -98,7 +98,7 @@ sub arpnip { foreach my $line (@lines) { if ($line =~ $linereg) { my ($ip, $mac) = ($1, $2); - if ($ip !~ m/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) { + if ($ip !~ m/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) { foreach my $name (@names) { if ($name =~ qr/name\s([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\s([\w-]*)/x) { if ($ip eq $2) { @@ -107,7 +107,7 @@ sub arpnip { } } } - if ($ip =~ m/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) { + if ($ip =~ m/^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/) { push @arpentries, { mac => $mac, ip => $ip }; } } diff --git a/lib/App/Netdisco/Util/Device.pm b/lib/App/Netdisco/Util/Device.pm index a4b481fe..e0fbd211 100644 --- a/lib/App/Netdisco/Util/Device.pm +++ b/lib/App/Netdisco/Util/Device.pm @@ -10,7 +10,7 @@ our @EXPORT_OK = qw/ get_device delete_device renumber_device - match_devicetype + match_to_setting is_discoverable is_discoverable_now is_arpnipable is_arpnipable_now is_macsuckable is_macsuckable_now @@ -100,8 +100,8 @@ sub delete_device { =head2 renumber_device( $current_ip, $new_ip ) Will update all records in Netdisco referring to the device with -C<$current_ip> to use C<$new_ip> instead, followed by renumbering the device -iteself. +C<$current_ip> to use C<$new_ip> instead, followed by renumbering the +device itself. Returns true if the transaction completes, else returns false. @@ -120,7 +120,7 @@ sub renumber_device { schema('netdisco')->resultset('UserLog')->create({ username => session('logged_in_user'), userip => scalar eval {request->remote_address}, - event => (sprintf "Renumber device %s to %s", $device->ip, $new_ip), + event => (sprintf "Renumber device %s to %s", $ip, $new_ip), }); $happy = 1; @@ -129,7 +129,7 @@ sub renumber_device { return $happy; } -=head2 match_devicetype( $type, $setting_name ) +=head2 match_to_setting( $type, $setting_name ) Given a C<$type> (which may be any text value), returns true if any of the list of regular expressions in C<$setting_name> is matched, otherwise returns @@ -137,7 +137,7 @@ false. =cut -sub match_devicetype { +sub match_to_setting { my ($type, $setting_name) = @_; return 0 unless $type and $setting_name; return (scalar grep {$type =~ m/$_/} @@ -146,7 +146,7 @@ sub match_devicetype { sub _bail_msg { debug $_[0]; return 0; } -=head2 is_discoverable( $ip, $device_type? ) +=head2 is_discoverable( $ip, [$device_type, \@device_capabilities]? ) Given an IP address, returns C if Netdisco on this host is permitted by the local configuration to discover the device. @@ -154,20 +154,32 @@ the local configuration to discover the device. The configuration items C and C are checked against the given IP. -If C<$device_type> is also given, then C will also be -checked. +If C<$device_type> is also given, then C will be checked. +Also respects C and C if either are set to +false. Returns false if the host is not permitted to discover the target device. =cut sub is_discoverable { - my ($ip, $remote_type) = @_; + my ($ip, $remote_type, $remote_cap) = @_; my $device = get_device($ip) or return 0; + $remote_type ||= ''; + $remote_cap ||= []; - if (match_devicetype($remote_type, 'discover_no_type')) { - return _bail_msg("is_discoverable: $device matched discover_no_type"); - } + return _bail_msg("is_discoverable: $device matches wap_platforms but discover_waps is not enabled") + if ((not setting('discover_waps')) and + (match_to_setting($remote_type, 'wap_platforms') or + scalar grep {match_to_setting($_, 'wap_capabilities')} @$remote_cap)); + + return _bail_msg("is_discoverable: $device matches phone_platforms but discover_phones is not enabled") + if ((not setting('discover_phones')) and + (match_to_setting($remote_type, 'phone_platforms') or + scalar grep {match_to_setting($_, 'phone_capabilities')} @$remote_cap)); + + return _bail_msg("is_discoverable: $device matched discover_no_type") + if (match_to_setting($remote_type, 'discover_no_type')); return _bail_msg("is_discoverable: $device matched discover_no") if check_acl_no($device, 'discover_no'); diff --git a/lib/App/Netdisco/Util/Port.pm b/lib/App/Netdisco/Util/Port.pm index 62b72f8e..383e893b 100644 --- a/lib/App/Netdisco/Util/Port.pm +++ b/lib/App/Netdisco/Util/Port.pm @@ -105,7 +105,8 @@ sub port_reconfig_check { # uplink check return "forbidden: port [$name] on [$ip] is an uplink" - if $port->remote_type and not $has_phone and not setting('portctl_uplinks'); + if ($port->is_uplink or $port->remote_type) + and not $has_phone and not setting('portctl_uplinks'); # phone check return "forbidden: port [$name] on [$ip] is a phone" @@ -215,18 +216,10 @@ sub is_vlan_interface { Returns true if the C<$port> L object has a phone connected. -This uses a simple check on the I of the remote connected device, and -therefore might sometimes return a false-negative result. - =cut sub port_has_phone { - my $port = shift; - - my $has_phone = ($port->remote_type - and $port->remote_type =~ /ip.phone/i) ? 1 : 0; - - return $has_phone; + return (shift)->with_properties->remote_is_phone; } 1; diff --git a/lib/App/Netdisco/Web/Device.pm b/lib/App/Netdisco/Web/Device.pm index e4cfdd00..010ed550 100644 --- a/lib/App/Netdisco/Web/Device.pm +++ b/lib/App/Netdisco/Web/Device.pm @@ -7,7 +7,7 @@ use Dancer::Plugin::Auth::Extensible; use URI (); use URL::Encode 'url_params_mixed'; -use App::Netdisco::Util::Device 'match_devicetype'; +use App::Netdisco::Util::Device 'match_to_setting'; # build view settings for port connected nodes and devices set('connected_properties' => [ @@ -20,7 +20,7 @@ hook 'before_template' => sub { my $tokens = shift; # allow checking of discoverability of remote connected device - $tokens->{has_snmp} = sub { not match_devicetype(shift, 'discover_no_type') }; + $tokens->{has_snmp} = sub { not match_to_setting(shift, 'discover_no_type') }; my $defaults = var('sidebar_defaults')->{'device_ports'} or return; diff --git a/lib/App/Netdisco/Web/GenericReport.pm b/lib/App/Netdisco/Web/GenericReport.pm index 800fa2ef..6e2f5855 100644 --- a/lib/App/Netdisco/Web/GenericReport.pm +++ b/lib/App/Netdisco/Web/GenericReport.pm @@ -9,7 +9,7 @@ use App::Netdisco::Web::Plugin; use Path::Class 'file'; use Safe; -use vars qw/$config @data/; +our ($config, @data); foreach my $report (@{setting('reports')}) { my $r = $report->{tag}; diff --git a/lib/App/Netdisco/Web/Plugin/AdminTask/UndiscoveredNeighbors.pm b/lib/App/Netdisco/Web/Plugin/AdminTask/UndiscoveredNeighbors.pm index d409d3cb..293d2525 100644 --- a/lib/App/Netdisco/Web/Plugin/AdminTask/UndiscoveredNeighbors.pm +++ b/lib/App/Netdisco/Web/Plugin/AdminTask/UndiscoveredNeighbors.pm @@ -27,6 +27,8 @@ get '/ajax/content/admin/undiscoveredneighbors' => require_role admin => sub { # create a new row object to avoid hitting the DB in get_device() my $dev = schema('netdisco')->resultset('Device')->new({ip => $r->{remote_ip}}); next unless is_discoverable( $dev, $r->{remote_type} ); + next if (not setting('discover_waps')) and $r->{remote_is_wap}; + next if (not setting('discover_phones')) and $r->{remote_is_phone}; push @discoverable_results, $r; } return unless scalar @discoverable_results; diff --git a/lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm b/lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm index bc1e3aec..bb63739e 100644 --- a/lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm +++ b/lib/App/Netdisco/Web/Plugin/Device/Neighbors.pm @@ -125,18 +125,13 @@ sub make_link_infostring { (my $left_name = lc($link->{left_dns} || $link->{left_name} || $link->{left_ip})) =~ s/$domain$//; (my $right_name = lc($link->{right_dns} || $link->{right_name} || $link->{right_ip})) =~ s/$domain$//; - if ($link->{aggports} == 1) { - return sprintf '%s:%s (%s)
%s:%s (%s)', - $left_name, $link->{left_port}->[0], - ($link->{left_descr}->[0] || 'no description'), - $right_name, $link->{right_port}->[0], - ($link->{right_descr}->[0] || 'no description'); - } - else { - return sprintf '%s:(%s)
%s:(%s)', - $left_name, join(',', @{$link->{left_port}}), - $right_name, join(',', @{$link->{right_port}}); - } + my @zipped = List::MoreUtils::zip6 + @{$link->{left_port}}, @{$link->{left_descr}}, + @{$link->{right_port}}, @{$link->{right_descr}}; + + return join '

', map { sprintf '%s:%s (%s)
%s:%s (%s)', + $left_name, $_->[0], ($_->[1] || 'no description'), + $right_name, $_->[2], ($_->[3] || 'no description') } @zipped; } ajax '/ajax/data/device/netmap' => require_login sub { @@ -179,17 +174,6 @@ ajax '/ajax/data/device/netmap' => require_login sub { ]) : ()) }, { result_class => 'DBIx::Class::ResultClass::HashRefInflator' }); - if ($vlan) { - $links = $links->search({ - -or => [ - { 'left_vlans.vlan' => $vlan }, - { 'right_vlans.vlan' => $vlan }, - ], - }, { - join => [qw/left_vlans right_vlans/], - }); - } - while (my $link = $links->next) { push @{$data{'links'}}, { FROMID => $link->{left_ip}, @@ -217,10 +201,19 @@ ajax '/ajax/data/device/netmap' => require_login sub { join => 'throughput', })->with_times; + # filter by vlan for all or neighbors only + if ($vlan) { + $devices = $devices->search( + { 'vlans.vlan' => $vlan }, + { join => 'vlans' } + ); + } + DEVICE: while (my $device = $devices->next) { - # if in neighbors or vlan mode then use %ok_dev to filter - next DEVICE if (($mapshow eq 'neighbors') or $vlan) - and (not $ok_dev{$device->ip}); + # if in neighbors mode then use %ok_dev to filter + next DEVICE if ($device->ip ne $qdev->ip) + and ($mapshow eq 'neighbors') + and (not $ok_dev{$device->ip}); # showing only neighbors but no link # if location picked then filter next DEVICE if ((scalar @lgrplist) and ((!defined $device->location) diff --git a/lib/App/Netdisco/Web/Plugin/Device/Ports.pm b/lib/App/Netdisco/Web/Plugin/Device/Ports.pm index b5276052..be5f98ea 100644 --- a/lib/App/Netdisco/Web/Plugin/Device/Ports.pm +++ b/lib/App/Netdisco/Web/Plugin/Device/Ports.pm @@ -25,25 +25,6 @@ get '/ajax/content/device/ports' => require_login sub { if ($f) { if (($prefer eq 'vlan') or (not $prefer and $f =~ m/^\d+$/)) { return unless $f =~ m/^\d+$/; - - if (param('invert')) { - $set = $set->search({ - 'me.vlan' => { '!=' => $f }, - 'port_vlans.vlan' => [ - '-or' => { '!=' => $f }, { '=' => undef } - ], - }, { join => 'port_vlans' }); - } - else { - $set = $set->search({ - -or => { - 'me.vlan' => $f, - 'port_vlans.vlan' => $f, - }, - }, { join => 'port_vlans' }); - } - - return unless $set->count; } else { if (param('partial')) { @@ -120,7 +101,7 @@ get '/ajax/content/device/ports' => require_login sub { # now begin to join tables depending on the selected columns/options # get vlans on the port - # leave this query dormant (lazy) unless c_vmember is set + # leave this query dormant (lazy) unless c_vmember is set or vlan filtering my $vlans = $set->search({}, { select => [ 'port', @@ -131,7 +112,7 @@ get '/ajax/content/device/ports' => require_login sub { group_by => 'me.port', }); - if (param('c_vmember')) { + if (param('c_vmember') or ($prefer eq 'vlan') or (not $prefer and $f =~ m/^\d+$/)) { $vlans = { map {( $_->port => { # DBIC smart enough to work out this should be an arrayref :) @@ -194,13 +175,37 @@ get '/ajax/content/device/ports' => require_login sub { # also get remote LLDP inventory if asked for $set = $set->with_remote_inventory if param('n_inventory'); - # sort ports (empty set would be a 'no records' msg) - my $results = [ sort { &App::Netdisco::Util::Web::sort_port($a->port, $b->port) } $set->all ]; - return unless scalar @$results; + # run query + my @results = $set->all; + + # filter for tagged vlan using existing agg query, + # which is better than join inflation + if (($prefer eq 'vlan') or (not $prefer and $f =~ m/^\d+$/)) { + if (param('invert')) { + @results = grep { + (!defined $_->vlan or $_->vlan ne $f) + and + (0 == scalar grep {defined and $_ ne $f} @{ $vlans->{$_->port}->{vlan_set} }) + } @results; + } + else { + @results = grep { + ($_->vlan eq $f) + or + (scalar grep {defined and $_ eq $f} @{ $vlans->{$_->port}->{vlan_set} }) + } @results; + } + } + + # sort ports + @results = sort { &App::Netdisco::Util::Web::sort_port($a->port, $b->port) } @results; + + # empty set would be a 'no records' msg + return unless scalar @results; if (request->is_ajax) { template 'ajax/device/ports.tt', { - results => $results, + results => \@results, nodes => $nodes_name, ips => $ips_name, device => $device, @@ -210,7 +215,7 @@ get '/ajax/content/device/ports' => require_login sub { else { header( 'Content-Type' => 'text/comma-separated-values' ); template 'ajax/device/ports_csv.tt', { - results => $results, + results => \@results, nodes => $nodes_name, ips => $ips_name, device => $device, diff --git a/lib/App/Netdisco/Web/Plugin/Device/Vlans.pm b/lib/App/Netdisco/Web/Plugin/Device/Vlans.pm new file mode 100644 index 00000000..b5d504a9 --- /dev/null +++ b/lib/App/Netdisco/Web/Plugin/Device/Vlans.pm @@ -0,0 +1,36 @@ +package App::Netdisco::Web::Plugin::Device::Vlans; + +use strict; +use warnings; + +use Dancer ':syntax'; +use Dancer::Plugin::Ajax; +use Dancer::Plugin::DBIC; +use Dancer::Plugin::Auth::Extensible; + +use App::Netdisco::Web::Plugin; + +register_device_tab({ tag => 'vlans', label => 'VLANs', provides_csv => 1 }); + +get '/ajax/content/device/vlans' => require_login sub { + my $q = param('q'); + + my $device = schema('netdisco')->resultset('Device') + ->search_for_device($q) or send_error('Bad device', 400); + my @results = $device->vlans->search( {}, { order_by => 'vlan' } )->hri->all; + + return unless scalar @results; + + if (request->is_ajax) { + my $json = to_json( \@results ); + template 'ajax/device/vlans.tt', { results => $json }, + { layout => undef }; + } + else { + header( 'Content-Type' => 'text/comma-separated-values' ); + template 'ajax/device/vlans_csv.tt', { results => \@results }, + { layout => undef }; + } +}; + +true; diff --git a/lib/App/Netdisco/Web/Plugin/Report/IpInventory.pm b/lib/App/Netdisco/Web/Plugin/Report/IpInventory.pm index 315db783..0fc809f0 100644 --- a/lib/App/Netdisco/Web/Plugin/Report/IpInventory.pm +++ b/lib/App/Netdisco/Web/Plugin/Report/IpInventory.pm @@ -110,12 +110,12 @@ get '/ajax/content/report/ipinventory' => require_login sub { 'ip', 'mac', 'dns', 'time_last', 'time_first', 'active', 'node', 'age' ], - order_by => [{-asc => 'ip'}, {-desc => 'active'}], + order_by => [{-asc => 'ip'}, {-desc => 'active'}, {-asc => 'node'}], } )->as_query; my $rs; - if ( $start && $end ) { + if ( $start and $end ) { $start = $start . ' 00:00:00'; $end = $end . ' 23:59:59'; diff --git a/lib/App/Netdisco/Web/Plugin/Search/Node.pm b/lib/App/Netdisco/Web/Plugin/Search/Node.pm index 45394e2c..6b74c86d 100644 --- a/lib/App/Netdisco/Web/Plugin/Search/Node.pm +++ b/lib/App/Netdisco/Web/Plugin/Search/Node.pm @@ -27,35 +27,48 @@ ajax '/ajax/content/search/node' => require_login sub { my @active = (param('archived') ? () : (-bool => 'active')); my (@times, @wifitimes, @porttimes); - if ($start and $end) { + if ( $start and $end ) { $start = $start . ' 00:00:00'; $end = $end . ' 23:59:59'; + if ($agenot) { @times = (-or => [ - time_first => [ { '<', $start }, undef ], - time_last => { '>', $end }, + time_first => [ undef ], + time_last => [ { '<', $start }, { '>', $end } ] ]); @wifitimes = (-or => [ - time_last => { '<', $start }, - time_last => { '>', $end }, + time_last => [ undef ], + time_last => [ { '<', $start }, { '>', $end } ], ]); @porttimes = (-or => [ - creation => { '<', $start }, - creation => { '>', $end }, + creation => [ undef ], + creation => [ { '<', $start }, { '>', $end } ] ]); } else { - @times = (-and => [ - time_first => { '>=', $start }, - time_last => { '<=', $end }, + @times = (-or => [ + -and => [ + time_first => undef, + time_last => undef, + ], + -and => [ + time_last => { '>=', $start }, + time_last => { '<=', $end }, + ], ]); - @wifitimes = (-and => [ - time_last => { '>=', $start }, - time_last => { '<=', $end }, + @wifitimes = (-or => [ + time_last => undef, + -and => [ + time_last => { '>=', $start }, + time_last => { '<=', $end }, + ], ]); - @porttimes = (-and => [ - creation => { '>=', $start }, - creation => { '<=', $end }, + @porttimes = (-or => [ + creation => undef, + -and => [ + creation => { '>=', $start }, + creation => { '<=', $end }, + ], ]); } } diff --git a/lib/App/Netdisco/Web/TypeAhead.pm b/lib/App/Netdisco/Web/TypeAhead.pm index 3994302e..40c7da62 100644 --- a/lib/App/Netdisco/Web/TypeAhead.pm +++ b/lib/App/Netdisco/Web/TypeAhead.pm @@ -49,7 +49,7 @@ ajax '/ajax/data/port/typeahead' => require_login sub { if $port; my $results = [ - map {{ label => (sprintf "%s (%s)", $_->port, $_->name), value => $_->port }} + map {{ label => (sprintf "%s (%s)", $_->port, ($_->name || '')), value => $_->port }} sort { &App::Netdisco::Util::Web::sort_port($a->port, $b->port) } $set->all ]; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 570b951b..a546a58d 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -45,7 +45,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { # only enqueue if device is not already discovered, # discover_* config permits the discovery foreach my $neighbor (@to_discover) { - my ($ip, $remote_type, $remote_id) = @$neighbor; + my ($ip, $remote_id) = @$neighbor; if ($seen_ip{ $ip }++) { debug sprintf ' queue - skip: IP %s is already queued from %s', @@ -63,13 +63,6 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my $newdev = get_device($ip); next if $newdev->in_storage; - if (not is_discoverable($newdev, $remote_type)) { - debug sprintf - ' queue - skip: %s of type [%s] excluded by discover_* config', - $ip, ($remote_type || ''); - next; - } - # risk of things going wrong...? # https://quickview.cloudapps.cisco.com/quickview/bug/CSCur12254 @@ -114,6 +107,7 @@ sub store_neighbors { or return (); # already checked! # first allow any manually configured topology to be set + # and do this before we cache the rows in vars->{'device_ports'} set_manual_topology($device); if (!defined $snmp->has_topo) { @@ -126,6 +120,12 @@ sub store_neighbors { my $c_port = $snmp->c_port; my $c_id = $snmp->c_id; my $c_platform = $snmp->c_platform; + my $c_cap = $snmp->c_cap; + + # cache the device ports to save hitting the database for many single rows + vars->{'device_ports'} = + { map {($_->port => $_)} $device->ports->reset->all }; + my $device_ports = vars->{'device_ports'}; # v4 and v6 neighbor tables my $c_ip = ($snmp->c_ip || {}); @@ -144,12 +144,12 @@ sub store_neighbors { next; } - my $port = $interfaces->{ $c_if->{$entry} }; - my $portrow = schema('netdisco')->resultset('DevicePort') - ->single({ip => $device->ip, port => $port}); + # WRT #475 this is SAFE because we check against known ports below + my $port = $interfaces->{ $c_if->{$entry} } or next; + my $portrow = $device_ports->{$port}; if (!defined $portrow) { - info sprintf ' [%s] neigh - local port %s not in database!', + debug sprintf ' [%s] neigh - local port %s already skipped, ignoring', $device->ip, $port; next; } @@ -161,7 +161,7 @@ sub store_neighbors { } if ($portrow->manual_topo) { - info sprintf ' [%s] neigh - %s has manually defined topology', + debug sprintf ' [%s] neigh - %s has manually defined topology', $device->ip, $port; next; } @@ -170,12 +170,13 @@ sub store_neighbors { my $remote_port = undef; my $remote_type = Encode::decode('UTF-8', $c_platform->{$entry} || ''); my $remote_id = Encode::decode('UTF-8', $c_id->{$entry}); + my $remote_cap = $c_cap->{$entry} || []; next unless $remote_ip; my $r_netaddr = NetAddr::IP::Lite->new($remote_ip); if ($r_netaddr and ($r_netaddr->addr ne $remote_ip)) { - info sprintf ' [%s] neigh - IP on %s: using %s as canonical form of %s', + debug sprintf ' [%s] neigh - IP on %s: using %s as canonical form of %s', $device->ip, $port, $r_netaddr->addr, $remote_ip; $remote_ip = $r_netaddr->addr; } @@ -189,7 +190,7 @@ sub store_neighbors { if ($remote_id) { my $devices = schema('netdisco')->resultset('Device'); my $neigh = $devices->single({name => $remote_id}); - info sprintf + debug sprintf ' [%s] neigh - bad address %s on port %s, searching for %s instead', $device->ip, $remote_ip, $port, $remote_id; @@ -207,7 +208,7 @@ sub store_neighbors { (my $tmpid = $remote_id) =~ s/.*\(([0-9a-f]{6})-([0-9a-f]{6})\).*/$1$2/; my $mac = NetAddr::MAC->new(mac => $tmpid); if ($mac and not $mac->errstr) { - info sprintf + debug sprintf ' [%s] neigh - trying to find neighbor %s by MAC %s', $device->ip, $remote_id, $mac->as_ieee; $neigh = $devices->single({mac => $mac->as_ieee}); @@ -221,17 +222,17 @@ sub store_neighbors { if ($neigh) { $remote_ip = $neigh->ip; - info sprintf ' [%s] neigh - found %s with IP %s', + debug sprintf ' [%s] neigh - found %s with IP %s', $device->ip, $remote_id, $remote_ip; } else { - info sprintf ' [%s] neigh - could not find %s, skipping', + debug sprintf ' [%s] neigh - could not find %s, skipping', $device->ip, $remote_id; next; } } else { - info sprintf ' [%s] neigh - skipping unuseable address %s on port %s', + debug sprintf ' [%s] neigh - skipping unuseable address %s on port %s', $device->ip, $remote_ip, $port; next; } @@ -240,7 +241,15 @@ sub store_neighbors { # what we came here to do.... discover the neighbor debug sprintf ' [%s] neigh - %s with ID [%s] on %s', $device->ip, $remote_ip, ($remote_id || ''), $port; - push @to_discover, [$remote_ip, $remote_type, $remote_id]; + + if (is_discoverable($remote_ip, $remote_type, $remote_cap)) { + push @to_discover, [$remote_ip, $remote_id]; + } + else { + debug sprintf + ' [%s] neigh - skip: %s of type [%s] excluded by discover_* config', + $device->ip, $remote_ip, ($remote_type || ''); + } $remote_port = $c_port->{$entry}; if (defined $remote_port) { @@ -248,18 +257,18 @@ sub store_neighbors { $remote_port =~ s/[^\d\s\/\.,()\w:-]+//gi; } else { - info sprintf ' [%s] neigh - no remote port found for port %s at %s', + debug sprintf ' [%s] neigh - no remote port found for port %s at %s', $device->ip, $port, $remote_ip; } - $portrow->update({ + $portrow = $portrow->update({ remote_ip => $remote_ip, remote_port => $remote_port, remote_type => $remote_type, remote_id => $remote_id, is_uplink => \"true", manual_topo => \"false", - }); + })->discard_changes(); # update master of our aggregate to be a neighbor of # the master on our peer device (a lot of iffs to get there...). diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm index a12d6b3d..25934ec5 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm @@ -22,8 +22,8 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my $eigrp_peers = $snmp->eigrp_peers || {}; return Status->info(" [$device] neigh - no BGP, OSPF, or EIGRP peers") - unless ((scalar values %$ospf_peers) or (scalar values %$bgp_peers) - or (scalar values %$eigrp_peers)); + unless ((scalar values %$ospf_peers) or (scalar values %$ospf_routers) + or (scalar values %$bgp_peers) or (scalar values %$eigrp_peers)); my $count = 0; foreach my $ip ((values %$ospf_peers), (values %$ospf_routers), diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm b/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm index c355224e..59a9d81b 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm @@ -32,6 +32,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { }; } + # cache the device ports to save hitting the database for many single rows + my $device_ports = vars->{'device_ports'} + || { map {($_->port => $_)} $device->ports->all }; + my $interfaces = $snmp->interfaces; my $p_ifindex = $snmp->peth_port_ifindex; my $p_admin = $snmp->peth_port_admin; @@ -42,11 +46,16 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { # build device port power info suitable for DBIC my @portpower; foreach my $entry (keys %$p_ifindex) { - my $port = $interfaces->{ $p_ifindex->{$entry} }; - next unless $port; + # WRT #475 this is SAFE because we check against known ports below + my $port = $interfaces->{ $p_ifindex->{$entry} } or next; + + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] power - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } my ($module) = split m/\./, $entry; - push @portpower, { port => $port, module => $module, diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm index 1680feac..5803cc6c 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/PortProperties.pm @@ -8,7 +8,7 @@ use App::Netdisco::Transport::SNMP (); use Dancer::Plugin::DBIC 'schema'; use Encode; -use App::Netdisco::Util::Device 'match_devicetype'; +use App::Netdisco::Util::Device 'match_to_setting'; register_worker({ phase => 'main', driver => 'snmp' }, sub { my ($job, $workerconf) = @_; @@ -21,10 +21,20 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my $interfaces = $snmp->interfaces || {}; my %properties = (); + # cache the device ports to save hitting the database for many single rows + my $device_ports = vars->{'device_ports'} + || { map {($_->port => $_)} $device->ports->all }; + my $raw_speed = $snmp->i_speed_raw || {}; foreach my $idx (keys %$raw_speed) { my $port = $interfaces->{$idx} or next; + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] properties/speed - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + $properties{ $port }->{raw_speed} = $raw_speed->{$idx}; } @@ -32,6 +42,12 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { foreach my $idx (keys %$err_cause) { my $port = $interfaces->{$idx} or next; + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] properties/errdis - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + $properties{ $port }->{error_disable_cause} = $err_cause->{$idx}; } @@ -39,6 +55,12 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { foreach my $idx (keys %$faststart) { my $port = $interfaces->{$idx} or next; + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] properties/faststart - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + $properties{ $port }->{faststart} = $faststart->{$idx}; } @@ -54,17 +76,22 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { foreach my $idx (keys %$c_if) { my $port = $interfaces->{ $c_if->{$idx} } or next; + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] properties/lldpcap - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } my $remote_cap = $c_cap->{$idx} || []; my $remote_type = Encode::decode('UTF-8', $c_platform->{$idx} || ''); $properties{ $port }->{remote_is_wap} = 'true' - if scalar grep {match_devicetype($_, 'wap_capabilities')} @$remote_cap - or match_devicetype($remote_type, 'wap_platforms'); + if scalar grep {match_to_setting($_, 'wap_capabilities')} @$remote_cap + or match_to_setting($remote_type, 'wap_platforms'); $properties{ $port }->{remote_is_phone} = 'true' - if scalar grep {match_devicetype($_, 'phone_capabilities')} @$remote_cap - or match_devicetype($remote_type, 'phone_platforms'); + if scalar grep {match_to_setting($_, 'phone_capabilities')} @$remote_cap + or match_to_setting($remote_type, 'phone_platforms'); next unless scalar grep {defined && m/^inventory$/} @{ $rem_media_cap->{$idx} }; @@ -74,17 +101,28 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { $properties{ $port }->{remote_serial} = $rem_serial->{ $idx }; } + foreach my $idx (keys %$interfaces) { + my $port = $interfaces->{$idx} or next; + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] properties/ifindex - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + + $properties{ $port }->{ifindex} = $idx; + } + return Status->info(" [$device] no port properties to record") unless scalar keys %properties; schema('netdisco')->txn_do(sub { my $gone = $device->properties_ports->delete; - debug sprintf ' [%s] props - removed %d ports with properties', + debug sprintf ' [%s] properties - removed %d ports with properties', $device->ip, $gone; $device->properties_ports->populate( [map {{ port => $_, %{ $properties{$_} } }} keys %properties] ); - return Status->info(sprintf ' [%s] props - added %d new port properties', + return Status->info(sprintf ' [%s] properties - added %d new port properties', $device->ip, scalar keys %properties); }); }); diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm index f2537ca7..95d2340d 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm @@ -151,7 +151,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub { if (defined $snmp->snmpEngineTime) { $dev_uptime_wrapped = int( $snmp->snmpEngineTime * 100 / 2**32 ); if ($dev_uptime_wrapped > 0) { - info sprintf ' [%s] interface - device uptime wrapped %d times - correcting', + debug sprintf ' [%s] interfaces - device uptime wrapped %d times - correcting', $device->ip, $dev_uptime_wrapped; $device->uptime( $dev_uptime + $dev_uptime_wrapped * 2**32 ); } @@ -190,7 +190,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub { my $lc = $i_lastchange->{$entry} || 0; if (not $dev_uptime_wrapped and $lc > $dev_uptime) { - info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting', + debug sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting', $device->ip, $port; $device->uptime( $dev_uptime + 2**32 ); $dev_uptime_wrapped = 1; @@ -238,6 +238,7 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub { # must do this after building %interfaces so that we can set is_master foreach my $sidx (keys %$agg_ports) { my $slave = $interfaces->{$sidx} or next; + next unless defined $agg_ports->{$sidx}; # slave without a master?! my $master = $interfaces->{ $agg_ports->{$sidx} } or next; next unless exists $interfaces{$slave} and exists $interfaces{$master}; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm b/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm index 0445661e..70bf26d2 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm @@ -33,6 +33,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { }; } + # cache the device ports to save hitting the database for many single rows + my $device_ports = vars->{'device_ports'} + || { map {($_->port => $_)} $device->ports->all }; + my $i_vlan = $snmp->i_vlan; my $i_vlan_membership = $snmp->i_vlan_membership; my $i_vlan_type = $snmp->i_vlan_type; @@ -42,8 +46,13 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my @portvlans = (); foreach my $entry (keys %$i_vlan_membership) { my %port_vseen = (); - my $port = $interfaces->{$entry}; - next unless defined $port; + my $port = $interfaces->{$entry} or next; + + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] vlans - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } my $type = $i_vlan_type->{$entry}; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm index a86640af..153b736e 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm @@ -18,6 +18,10 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my $ssidlist = $snmp->i_ssidlist; return unless scalar keys %$ssidlist; + # cache the device ports to save hitting the database for many single rows + my $device_ports = vars->{'device_ports'} + || { map {($_->port => $_)} $device->ports->all }; + my $interfaces = $snmp->interfaces; my $ssidbcast = $snmp->i_ssidbcast; my $ssidmac = $snmp->i_ssidmac; @@ -36,6 +40,12 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { next; } + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] wireless - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + push @ssids, { port => $port, ssid => $ssidlist->{$entry}, @@ -64,6 +74,12 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { next; } + if (!defined $device_ports->{$port}) { + debug sprintf ' [%s] wireless - local port %s already skipped, ignoring', + $device->ip, $port; + next; + } + push @channels, { port => $port, channel => $channel->{$entry}, diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm index b46e78a4..9ea9dfb9 100644 --- a/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm @@ -7,7 +7,7 @@ use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::Transport::SNMP (); use App::Netdisco::Util::Permission 'check_acl_no'; use App::Netdisco::Util::PortMAC 'get_port_macs'; -use App::Netdisco::Util::Device 'match_devicetype'; +use App::Netdisco::Util::Device 'match_to_setting'; use App::Netdisco::Util::Node 'check_mac'; use App::Netdisco::Util::SNMP 'snmp_comm_reindex'; use Dancer::Plugin::DBIC 'schema'; @@ -248,7 +248,6 @@ sub get_vlan_list { # check in use by a port on this device if (!$vlans{$vlan} && !setting('macsuck_all_vlans')) { - debug sprintf ' [%s] macsuck VLAN %s/%s - not in use by any port - skipping.', $device->ip, $vlan, $name; @@ -299,6 +298,8 @@ sub walk_fwtable { next; } + # WRT #475 this is SAFE because we check against known ports below + # but we do need the SNMP interface IDs to get the job done my $port = $interfaces->{$iid}; unless (defined $port) { @@ -318,6 +319,7 @@ sub walk_fwtable { # this uses the cached $ports resultset to limit hits on the db my $device_port = $device_ports->{$port}; + # WRT #475 ... see? :-) unless (defined $device_port) { debug sprintf ' [%s] macsuck %s - port %s is not in database - skipping.', @@ -340,7 +342,7 @@ sub walk_fwtable { # neighbors otherwise it would kill the DB with device lookups. my $neigh_cannot_macsuck = eval { # can fail check_acl_no(($device_port->neighbor || "0 but true"), 'macsuck_unsupported') || - match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') }; + match_to_setting($device_port->remote_type, 'macsuck_unsupported_type') }; if ($device_port->is_uplink) { if ($neigh_cannot_macsuck) { diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm index 44c7d5b5..454bc4c3 100644 --- a/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm @@ -41,11 +41,11 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { next unless defined $mac; # avoid null entries # there can be more rows in txrate than other tables - my $txrate = defined $txrates->[$#$txrates] + my $txrate = (ref $txrates and defined $txrates->[$#$txrates]) ? int($txrates->[$#$txrates]) : undef; - my $maxrate = defined $rates->[$#$rates] + my $maxrate = (ref $rates and defined $rates->[$#$rates]) ? int($rates->[$#$rates]) : undef; diff --git a/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm b/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm index 8f241bae..390d0962 100644 --- a/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm +++ b/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm @@ -1,5 +1,8 @@ package App::Netdisco::Worker::Plugin::MakeRancidConf; +use strict; +use warnings; + use Dancer ':syntax'; use Dancer::Plugin::DBIC; @@ -8,7 +11,7 @@ use aliased 'App::Netdisco::Worker::Status'; use Path::Class; use List::Util qw/pairkeys pairfirst/; -use File::Slurper 'write_text'; +use File::Slurper qw/read_lines write_text/; use App::Netdisco::Util::Permission 'check_acl_no'; register_worker({ phase => 'main' }, sub { @@ -16,14 +19,31 @@ register_worker({ phase => 'main' }, sub { my $config = setting('rancid') || {}; my $domain_suffix = setting('domain_suffix') || ''; - my $delimiter = $config->{delimiter} || ':'; + my $delimiter = $config->{delimiter} || ';'; my $down_age = $config->{down_age} || '1 day'; + my $default_group = $config->{default_group} || 'default'; - my $rancidhome = $config->{rancid_home} + my $rancidconf = $config->{rancid_conf} || '/etc/rancid'; + my $rancidcvsroot = $config->{rancid_cvsroot} || dir($ENV{NETDISCO_HOME}, 'rancid')->stringify; - mkdir $rancidhome if ! -d $rancidhome; - return Status->error("cannot create or see rancid home: $rancidhome") - if ! -d $rancidhome; + mkdir $rancidcvsroot if ! -d $rancidcvsroot; + return Status->error("cannot create or access rancid cvsroot: $rancidcvsroot") + if ! -d $rancidcvsroot; + + my $allowed_types = {}; + foreach my $type (qw/base conf/) { + my $type_file = file($rancidconf, "rancid.types.$type")->stringify; + debug sprintf("trying rancid configuration file %s\n", $type_file); + next unless -f $type_file; + my @lines = read_lines($type_file); + foreach my $line (@lines) { + next if $line =~ m/^(?:\#|\$)/; + $allowed_types->{$1} += 1 if $line =~ m/^([a-z0-9_\-]+);login;.*$/; + } + } + + return Status->error("You didn't have any device types configured in your rancid installation.") + if ! scalar keys %$allowed_types; my $devices = schema('netdisco')->resultset('Device')->search(undef, { '+columns' => { old => @@ -32,24 +52,37 @@ register_worker({ phase => 'main' }, sub { $config->{groups} ||= { default => 'any' }; $config->{vendormap} ||= {}; + $config->{excluded} ||= {}; + $config->{by_ip} ||= {}; + $config->{by_hostname} ||= {}; my $routerdb = {}; while (my $d = $devices->next) { - my $name = - check_acl_no($d, $config->{by_ip}) ? $d->ip : ($d->dns || $d->name); - $name =~ s/$domain_suffix$// - if check_acl_no($d, $config->{by_hostname}); + + if (check_acl_no($d, $config->{excluded})) { + debug " skipping $d: device excluded of export"; + next + } + + my $name = check_acl_no($d, $config->{by_ip}) ? $d->ip : ($d->dns || $d->name); + $name =~ s/$domain_suffix$// if check_acl_no($d, $config->{by_hostname}); my ($group) = - pairkeys pairfirst { check_acl_no($d, $b) } %{ $config->{groups} }; + (pairkeys pairfirst { check_acl_no($d, $b) } %{ $config->{groups} }) || $default_group; my ($vendor) = (pairkeys pairfirst { check_acl_no($d, $b) } %{ $config->{vendormap} }) || $d->vendor; - if ($vendor =~ m/(?:enterprises\.|netdisco)/) { + if (not ($name and $vendor)) { + debug " skipping $d: the name or vendor is not defined"; + next + } elsif ($vendor =~ m/(?:enterprises\.|netdisco)/) { debug " skipping $d with unresolved vendor: $vendor"; next; + } elsif (scalar keys %$allowed_types and !exists($allowed_types->{$vendor})) { + debug " skipping $d: $vendor doesn't exist in rancid's vendor list"; + next; } push @{$routerdb->{$group}}, @@ -58,12 +91,14 @@ register_worker({ phase => 'main' }, sub { } foreach my $group (keys %$routerdb) { - mkdir dir($rancidhome, $group)->stringify; - my $content = join "\n", @{$routerdb->{$group}}; - write_text(file($rancidhome, $group, 'router.db')->stringify, "${content}\n"); + mkdir dir($rancidcvsroot, $group)->stringify; + my $content = "#\n# Router list file for rancid group $group.\n"; + $content .= "# Generate automatically by App::Netdisco::Worker::Plugin::MakeRancidConf\n#\n"; + $content .= join "\n", sort @{$routerdb->{$group}}; + write_text(file($rancidcvsroot, $group, 'router.db')->stringify, "${content}\n"); } - return Status->done('Wrote RANCID configuration.'); + return Status->done('Wrote rancid configuration.'); }); true; @@ -72,17 +107,20 @@ true; =head1 NAME -MakeRancidConf - Generate RANCID Configuration +MakeRancidConf - Generate rancid Configuration =head1 INTRODUCTION -This worker will generate a RANCID configuration for all devices in Netdisco. +This worker will generate a rancid configuration for all devices in Netdisco. Optionally you can provide configuration to control the output, however the -defaults are sane, and will create one RANCID group called "C" which -contains all devices. Those devices not discovered successfully within the -past day will be marked as "down" for RANCID to skip. Configuration is saved -to the "rancid" subdirectory of Netdisco's home folder. +defaults are sane for rancid versions 3.x and will create one rancid group +called C which contains all devices. Those devices not discovered +successfully within the past day will be marked as C for rancid to skip. +Configuration is saved to the F<~/rancid> subdirectory of Netdisco's home folder. + +Note that this only generates the router.db files, you will still need to +configure rancid's F<.cloginrc> and schedule C to run. You could run this worker at 09:05 each day using the following configuration: @@ -90,95 +128,142 @@ You could run this worker at 09:05 each day using the following configuration: makerancidconf: when: '5 9 * * *' +Since MakeRancidConf is a worker module it can also be run via C: + + netdisco-do makerancidconf + =head1 CONFIGURATION Here is a complete example of the configuration, which must be called -"C". All keys are optional: +C. All keys are optional: rancid: - rancid_home: "$ENV{NETDISCO_HOME}/rancid" # default - down_age: '1 day' # default - delimiter: ':' # default + rancid_cvsroot: '$ENV{NETDISCO_HOME}/rancid' # default + rancid_conf: '/etc/rancid' # default + down_age: '1 day' # default + delimiter: ';' # default + default_group: 'default' # default + excluded: + excludegroup1: 'host_group1_acl' + excludegroup2: 'host_group2_acl' groups: - groupname1: 'host_group1_acl' - groupname2: 'host_group2_acl' + groupname1: 'host_group3_acl' + groupname2: 'host_group4_acl' vendormap: - vname1: 'host_group3_acl' - vname2: 'host_group4_acl' - by_ip: 'host_group5_acl' - by_hostname: 'host_group6_acl' + vname1: 'host_group5_acl' + vname2: 'host_group6_acl' + by_ip: 'host_group7_acl' + by_hostname: 'host_group8_acl' -Note that the default home for writing files is not "C" so -you may wish to set this (especially if migrating from the old +Note that the default directory for writing files is not F so +you may wish to set this in C, (especially if migrating from the old C script). -Any values above that are a Host Group ACL will take either a single item or -list of Network Identifiers or Device Properties. See the L -wiki page for full details. We advise you to use the "C" setting +wiki page for full details. We advise you to use the C setting and then refer to named entries in that, for example: host_groups: coredevices: '192.0.2.0/24' edgedevices: '172.16.0.0/16' - + grp-nxos: 'os:nx-os' + rancid: groups: core_devices: 'group:coredevices' edge_devices: 'group:edgedevices' + vendormap: + cisco-nx: 'group:grp-nxos' + by_ip: 'any' -=head2 C +Do not forget that rancid also needs configuring when adding a new group, +such as scheduling the group to run, adding it to F, setting up the +email config and creating the repository with C. -The location to write RANCID Group configuration files into. A subdirectory -for each Group will be created. +=head2 C + +The location where the rancid configuration (F and +F) is installed. It will be used to check the existance +of device types before exporting the devices to the rancid configuration. if no match +is found the device will not be added to rancid. + +=head2 C + +The location to write rancid group configuration files (F) into. A +subdirectory for each group will be created. =head2 C This should be the same or greater than the interval between regular discover jobs on your network. Devices which have not been discovered within this time -will be marked as "C" to RANCID. +will be marked as C to rancid. The format is any time interval known and understood by PostgreSQL, such as at -L. +L. =head2 C -Set this to the delimiter character if needed to be different from the -default. +Set this to the delimiter character for your F entries if needed to +be different from the default, the default is C<;>. + +=head2 C + +Put devices into this group if they do not match any other groups defined. + +=head2 C + +This dictionary defines a list of devices that you do not wish to export to +rancid configuration. + +The value should be a L +to select devices in the Netdisco database. =head2 C -This dictionary maps RANCID Group names with configuration which will match +This dictionary maps rancid group names with configuration which will match devices in the Netdisco database. -The left hand side (key) should be the RANCID group name, the right hand side +The left hand side (key) should be the rancid group name, the right hand side (value) should be a L to select devices in the Netdisco database. =head2 C -If the device Vendor in Netdisco is not the same as the RANCID vendor script, -configure a mapping here. +If the device vendor in Netdisco is not the same as the rancid vendor script or +device type, configure a mapping here. -The left hand side (key) should be the RANCID vendor, the right hand side +The left hand side (key) should be the rancid device type, the right hand side (value) should be a L to select devices in the Netdisco database. +Note that vendors might have a large array of operating systems which require +different rancid modules. Mapping operating systems to rancid device types is +a good solution to use the correct device type. Example: + + host_groups: + grp-ciscosb: 'os:ros' + + rancid: + vendormap: + cisco-sb: 'group:grp-ciscosb' + =head2 C L -to select devices which will be written to the RANCID config as an IP address, -instead of the DNS FQDN or SNMP host name. +to select devices which will be written to the rancid config as an IP address, +instead of the DNS FQDN or SNMP hostname. =head2 C L -to select devices which will have the unqualified host name written to the -RANCID config. This is done simply by stripping the C +to select devices which will have the unqualified hostname written to the +rancid config. This is done simply by stripping the C configuration setting from the device FQDN. =head1 SEE ALSO diff --git a/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm b/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm index 7b12ebea..5ccfaa2b 100644 --- a/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm +++ b/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm @@ -47,4 +47,3 @@ register_worker({ phase => 'main' }, sub { }); true; - diff --git a/lib/App/Netdisco/Worker/Plugin/PortControl.pm b/lib/App/Netdisco/Worker/Plugin/PortControl.pm index f767a874..a2607c9a 100644 --- a/lib/App/Netdisco/Worker/Plugin/PortControl.pm +++ b/lib/App/Netdisco/Worker/Plugin/PortControl.pm @@ -17,9 +17,9 @@ register_worker({ phase => 'check' }, sub { or return Status->error(sprintf "Unknown port name [%s] on device %s", $job->port, $job->device); - my $vlan_reconfig_check = vlan_reconfig_check(vars->{'port'}); - return Status->error("Cannot alter vlan: $vlan_reconfig_check") - if $vlan_reconfig_check; + my $port_reconfig_check = port_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter port: $port_reconfig_check") + if $port_reconfig_check; return Status->done('PortControl is able to run'); }); @@ -56,7 +56,7 @@ sub _action { my $rv = $snmp->set_i_up_admin($data, $iid); if (!defined $rv) { - return Status->error(sprintf 'Failed to set [%s] up_admin to [%s] on $device: %s', + return Status->error(sprintf "Failed to set [%s] up_admin to [%s] on $device: %s", $pn, $data, ($snmp->error || '')); } diff --git a/lib/App/Netdisco/Worker/Plugin/Power.pm b/lib/App/Netdisco/Worker/Plugin/Power.pm index 81099b48..b6789698 100644 --- a/lib/App/Netdisco/Worker/Plugin/Power.pm +++ b/lib/App/Netdisco/Worker/Plugin/Power.pm @@ -19,9 +19,9 @@ register_worker({ phase => 'check' }, sub { or return Status->error(sprintf "Unknown port name [%s] on device %s", $job->port, $job->device); - my $vlan_reconfig_check = vlan_reconfig_check(vars->{'port'}); - return Status->error("Cannot alter vlan: $vlan_reconfig_check") - if $vlan_reconfig_check; + my $port_reconfig_check = port_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter port: $port_reconfig_check") + if $port_reconfig_check; return Status->error("No PoE service on port [$pn] on device $device") unless vars->{'port'}->power; @@ -40,7 +40,7 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { # snmp connect using rw community my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) - or return Status->defer("failed to connect to $device to update vlan"); + or return Status->defer("failed to connect to $device to set power"); my $powerid = get_powerid($snmp, vars->{'port'}) or return Status->error("failed to get power ID for [$pn] from $device"); diff --git a/lib/App/Netdisco/Worker/Plugin/Show.pm b/lib/App/Netdisco/Worker/Plugin/Show.pm index 52a2ca55..6ec90b90 100644 --- a/lib/App/Netdisco/Worker/Plugin/Show.pm +++ b/lib/App/Netdisco/Worker/Plugin/Show.pm @@ -18,13 +18,14 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; $extra ||= 'interfaces'; my $class = undef; - ($class, $extra) = split(/::([^:]+)$/, $extra); - if ($class and $extra) { - $class = 'SNMP::Info::'.$class; - } - else { - $extra = $class; - undef $class; + my @values = split /::/, $extra; + $extra = pop @values; + if (scalar(@values)) { + $class = "SNMP::Info"; + foreach my $v (@values) { + last if ($v eq ''); + $class = $class.'::'.$v; + } } my $i = App::Netdisco::Transport::SNMP->reader_for($device, $class); diff --git a/share/config.yml b/share/config.yml index 5996d436..3334b4d0 100644 --- a/share/config.yml +++ b/share/config.yml @@ -88,6 +88,7 @@ web_plugins: - Device::Modules - Device::Neighbors - Device::Addresses + - Device::Vlans extra_web_plugins: [] sidebar_defaults: search_node: @@ -229,9 +230,9 @@ devices_no: [] devices_only: [] discover_no: [] discover_only: [] -discover_no_type: - - '(?i)phone' - - '(?i)(?:wap|wireless)' +discover_no_type: [] +discover_waps: true +discover_phones: false discover_min_age: 0 macsuck_no: [] macsuck_only: [] diff --git a/share/environments/deployment.yml b/share/environments/deployment.yml index cd792907..a1147c97 100644 --- a/share/environments/deployment.yml +++ b/share/environments/deployment.yml @@ -39,20 +39,18 @@ device_auth: # ¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸ +# +# SOME MORE INTERESTING SETTINGS WHERE THE DEFAULTS ARE PROBABLY OKAY +# # ¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸ -# --------------------------------------------------------------- -# OTHER INTERESTING SETTINGS WHERE THE DEFAULTS ARE PROBABLY OKAY -# --------------------------------------------------------------- - -# do not discover IP Phones or Wireless Access Points. +# discover Wireless Access Points, but not IP Phones # usually these are visible as device neighbors but don't support # SNMP, which just clogs up the job queue. # ``````````````````````````````````````````````````````````````` -#discover_no_type: -# - '(?i)phone' -# - '(?i)(?:wap|wireless)' +#discover_waps: true +#disover_phones: false # this is the schedule for automatically keeping netdisco up-to-date; # these are good defaults, so only uncomment if needing to change. diff --git a/share/schema_versions/App-Netdisco-DB-53-54-PostgreSQL.sql b/share/schema_versions/App-Netdisco-DB-53-54-PostgreSQL.sql new file mode 100644 index 00000000..64454a87 --- /dev/null +++ b/share/schema_versions/App-Netdisco-DB-53-54-PostgreSQL.sql @@ -0,0 +1,5 @@ +BEGIN; + +ALTER TABLE device_port_properties ADD COLUMN "ifindex" bigint; + +COMMIT; diff --git a/share/views/ajax/device/addresses.tt b/share/views/ajax/device/addresses.tt index 448bc385..9ab40efc 100644 --- a/share/views/ajax/device/addresses.tt +++ b/share/views/ajax/device/addresses.tt @@ -32,7 +32,7 @@ $(document).ready(function() { "render": function(data, type, row, meta) { return type === 'display' ? '' + he.encode(data || '') + '' : - data; + he.encode(data || ''); } }, { "data": 'name', diff --git a/share/views/ajax/device/details.tt b/share/views/ajax/device/details.tt index 6e6e611b..95720225 100644 --- a/share/views/ajax/device/details.tt +++ b/share/views/ajax/device/details.tt @@ -50,10 +50,12 @@ OS / Version - [% d.os | html_entity %] / - [% d.os_ver | html_entity %] + + [% d.os | html_entity %] + / + [% d.os_ver | html_entity %] diff --git a/share/views/ajax/device/ports.tt b/share/views/ajax/device/ports.tt index afd86b4f..e2498226 100644 --- a/share/views/ajax/device/ports.tt +++ b/share/views/ajax/device/ports.tt @@ -12,7 +12,15 @@ [% th_class = ' class="portsort"' %] [% END %] + [% IF item.name == 'c_neighbors' %] + [% IF params.c_nodes %] + Connected Nodes & Devices + [% ELSE %] + Connected Devices + [% END %] + [% ELSE %] [% item.label | html_entity %] + [% END %] [% END %] @@ -361,8 +369,8 @@ - [% END %] + [% END %] [% END %] diff --git a/share/views/ajax/device/vlans.tt b/share/views/ajax/device/vlans.tt new file mode 100644 index 00000000..aa8f795f --- /dev/null +++ b/share/views/ajax/device/vlans.tt @@ -0,0 +1,31 @@ + + + + + + + +
VLAN IDVLAN Name
+ + diff --git a/share/views/ajax/device/vlans_csv.tt b/share/views/ajax/device/vlans_csv.tt new file mode 100644 index 00000000..51bb160c --- /dev/null +++ b/share/views/ajax/device/vlans_csv.tt @@ -0,0 +1,10 @@ +[% USE CSV -%] +[% CSV.dump([ 'VLAN ID' 'VLAN Name' ]) %] + +[% FOREACH row IN results %] + [% mylist = [] %] + [% mylist.push(row.vlan) %] + [% mylist.push(row.description) %] + [% CSV.dump(mylist) %] + +[% END %] diff --git a/share/views/ajax/report/apclients.tt b/share/views/ajax/report/apclients.tt index c3fc8cd3..bcfef160 100644 --- a/share/views/ajax/report/apclients.tt +++ b/share/views/ajax/report/apclients.tt @@ -41,8 +41,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'ssid', diff --git a/share/views/ajax/report/apradiochannelpower.tt b/share/views/ajax/report/apradiochannelpower.tt index 3a283562..6faf2299 100644 --- a/share/views/ajax/report/apradiochannelpower.tt +++ b/share/views/ajax/report/apradiochannelpower.tt @@ -59,8 +59,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'port_name', diff --git a/share/views/ajax/report/duplexmismatch.tt b/share/views/ajax/report/duplexmismatch.tt index 17548fc3..7d36bfd2 100644 --- a/share/views/ajax/report/duplexmismatch.tt +++ b/share/views/ajax/report/duplexmismatch.tt @@ -27,8 +27,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'left_duplex', @@ -45,8 +45,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'right_duplex', diff --git a/share/views/ajax/report/halfduplex.tt b/share/views/ajax/report/halfduplex.tt index 4427ce99..8816d758 100644 --- a/share/views/ajax/report/halfduplex.tt +++ b/share/views/ajax/report/halfduplex.tt @@ -25,8 +25,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'name', diff --git a/share/views/ajax/report/ipinventory.tt b/share/views/ajax/report/ipinventory.tt index 7ff4a98f..bf48181f 100644 --- a/share/views/ajax/report/ipinventory.tt +++ b/share/views/ajax/report/ipinventory.tt @@ -1,7 +1,7 @@ - + diff --git a/share/views/ajax/report/nodesdiscovered.tt b/share/views/ajax/report/nodesdiscovered.tt index 98db813e..841d17f2 100644 --- a/share/views/ajax/report/nodesdiscovered.tt +++ b/share/views/ajax/report/nodesdiscovered.tt @@ -27,8 +27,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'remote_id', diff --git a/share/views/ajax/report/portadmindown.tt b/share/views/ajax/report/portadmindown.tt index 9a8c516c..207be63a 100644 --- a/share/views/ajax/report/portadmindown.tt +++ b/share/views/ajax/report/portadmindown.tt @@ -25,8 +25,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'description', diff --git a/share/views/ajax/report/portblocking.tt b/share/views/ajax/report/portblocking.tt index 5ff08dc9..7632eb3d 100644 --- a/share/views/ajax/report/portblocking.tt +++ b/share/views/ajax/report/portblocking.tt @@ -26,8 +26,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'description', diff --git a/share/views/ajax/report/portmultinodes.tt b/share/views/ajax/report/portmultinodes.tt index 8b45c411..30ccb6ad 100644 --- a/share/views/ajax/report/portmultinodes.tt +++ b/share/views/ajax/report/portmultinodes.tt @@ -26,8 +26,8 @@ $(document).ready(function() { "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'description', diff --git a/share/views/ajax/report/portvlanmismatch.tt b/share/views/ajax/report/portvlanmismatch.tt index 6f276177..80a17501 100644 --- a/share/views/ajax/report/portvlanmismatch.tt +++ b/share/views/ajax/report/portvlanmismatch.tt @@ -20,27 +20,27 @@ $(document).ready(function() { { "data": 'left_device', "render": function(data, type, row, meta) { - return '' + he.encode(data) + ''; } + return '' + he.encode(data || '') + ''; } }, { "data": 'left_port', "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; } + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'left_vlans' }, { "data": 'right_device', "render": function(data, type, row, meta) { - return '' + he.encode(data) + ''; } + return '' + he.encode(data || '') + ''; } }, { "data": 'right_port', "type": 'portsort', "render": function(data, type, row, meta) { return type === 'display' ? - '' + he.encode(data) + '' : - data; } + '' + he.encode(data || '') + '' : + he.encode(data || ''); } }, { "data": 'right_vlans' } diff --git a/share/views/ajax/search/port.tt b/share/views/ajax/search/port.tt index 25c1a1cc..99635b8d 100644 --- a/share/views/ajax/search/port.tt +++ b/share/views/ajax/search/port.tt @@ -4,7 +4,7 @@ - + diff --git a/share/views/ajax/search/port_csv.tt b/share/views/ajax/search/port_csv.tt index e9aa5db8..5ac669ee 100644 --- a/share/views/ajax/search/port_csv.tt +++ b/share/views/ajax/search/port_csv.tt @@ -1,5 +1,5 @@ [% USE CSV -%] -[% CSV.dump([ 'Name' 'Port' 'Description' 'Vlan' ]) %] +[% CSV.dump([ 'Name' 'Port' 'Description' 'VLAN' ]) %] [% FOREACH row IN results %] [% mylist = [] %] diff --git a/share/views/ajax/search/vlan.tt b/share/views/ajax/search/vlan.tt index aa75ce85..aa81060d 100644 --- a/share/views/ajax/search/vlan.tt +++ b/share/views/ajax/search/vlan.tt @@ -1,7 +1,7 @@
NodeIP Address MAC Address DNS Last UsedStatus Name PortVlanVLAN Speed Last Change
- + diff --git a/share/views/ajax/search/vlan_csv.tt b/share/views/ajax/search/vlan_csv.tt index 1d095856..82b0e33e 100644 --- a/share/views/ajax/search/vlan_csv.tt +++ b/share/views/ajax/search/vlan_csv.tt @@ -1,5 +1,5 @@ [% USE CSV -%] -[% CSV.dump([ 'Vlan' 'Device' 'Description' 'Model' 'OS' 'Vendor' ]) %] +[% CSV.dump([ 'VLAN' 'Device' 'Description' 'Model' 'OS' 'Vendor' ]) %] [% FOREACH row IN results %] [% mylist = [] %] diff --git a/share/views/ajax/statistics.tt b/share/views/ajax/statistics.tt index 524f390d..d8bd1d3b 100644 --- a/share/views/ajax/statistics.tt +++ b/share/views/ajax/statistics.tt @@ -57,7 +57,6 @@ - diff --git a/share/views/sidebar/report/ipinventory.tt b/share/views/sidebar/report/ipinventory.tt index c5c01e25..b8ddb524 100644 --- a/share/views/sidebar/report/ipinventory.tt +++ b/share/views/sidebar/report/ipinventory.tt @@ -46,7 +46,7 @@ rel="tooltip" data-placement="left" data-offset="5" data-title="Applies to IPv4 Only">
VlanVLAN Device Description Model
PostgreSQL [% stats.pg_ver | html_entity %]
Perl