diff --git a/Netdisco/Changes b/Netdisco/Changes index 781e3fff..36004b67 100644 --- a/Netdisco/Changes +++ b/Netdisco/Changes @@ -1,3 +1,9 @@ +2.029006 - + + [NEW FEATURES] + + * netdisco-sshcollector script to get ARP data on devices without SNMP (C. Ramseyer) + 2.029005 - 2014-08-13 [ENHANCEMENTS] diff --git a/Netdisco/MANIFEST b/Netdisco/MANIFEST index 8ef8ad38..55ae30c4 100644 --- a/Netdisco/MANIFEST +++ b/Netdisco/MANIFEST @@ -6,6 +6,7 @@ bin/netdisco-db-deploy bin/netdisco-deploy bin/netdisco-do bin/netdisco-rancid-export +bin/netdisco-sshcollector bin/netdisco-web bin/netdisco-web-fg Changes @@ -148,6 +149,9 @@ lib/App/Netdisco/Manual/Deployment.pod lib/App/Netdisco/Manual/Developing.pod lib/App/Netdisco/Manual/ReleaseNotes.pod lib/App/Netdisco/Manual/WritingPlugins.pod +lib/App/Netdisco/SSHCollector/Platform/ACE.pm +lib/App/Netdisco/SSHCollector/Platform/BigIP.pm +lib/App/Netdisco/SSHCollector/Platform/IOS.pm lib/App/Netdisco/Util/Daemon.pm lib/App/Netdisco/Util/Device.pm lib/App/Netdisco/Util/DNS.pm diff --git a/Netdisco/META.yml b/Netdisco/META.yml index 2d7e34e7..1d97f731 100644 --- a/Netdisco/META.yml +++ b/Netdisco/META.yml @@ -24,6 +24,7 @@ no_index: recommends: Graph: 0 GraphViz: 0 + Net::OpenSSH: 0 requires: Algorithm::Cron: 0.07 AnyEvent: 7.05 @@ -69,7 +70,6 @@ requires: Sereal: 0 Socket6: 0.23 Starman: 0.4008 - Sys::Proctitle: 0 Template: 2.24 Template::Plugin::CSV: 0.04 Template::Plugin::Number::Format: 1.02 diff --git a/Netdisco/Makefile.PL b/Netdisco/Makefile.PL index 8e818eb4..1e53763b 100644 --- a/Netdisco/Makefile.PL +++ b/Netdisco/Makefile.PL @@ -70,6 +70,7 @@ if ( $^O eq 'linux' ) { recommends 'Graph' => 0; recommends 'GraphViz' => 0; +recommends 'Net::OpenSSH' => 0; install_share 'share'; @@ -82,6 +83,7 @@ install_script 'bin/netdisco-daemon'; install_script 'bin/netdisco-web-fg'; install_script 'bin/netdisco-web'; install_script 'bin/netdisco-rancid-export'; +install_script 'bin/netdisco-sshcollector'; resources homepage => 'http://netdisco.org/', diff --git a/Netdisco/bin/netdisco-sshcollector b/Netdisco/bin/netdisco-sshcollector new file mode 100644 index 00000000..38747b1b --- /dev/null +++ b/Netdisco/bin/netdisco-sshcollector @@ -0,0 +1,268 @@ +#!/usr/bin/env perl + +# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4: + +=head1 NAME + +netdisco-sshcollector - Collect ARP data for Netdisco from devices without +full SNMP support + +=head1 DESCRIPTION + +Collects ARP data for Netdisco from devices without full SNMP support. +Currently, ARP tables can be retrieved from the following device classes: + +=over 4 + +=item * L - Cisco ACE (Application Control Engine) + +=item * L - F5 Networks BigIP + +=item * L - Cisco IOS + +=back + +The collected arp entries are then directly stored in the netdisco database. + +=head1 CONFIGURATION + +The following should go into your Netdisco 2 configuration file, "C<< +~/environments/deployment.yml >>" + +=over 4 + +=item C + +Data is collected from the machines specified in this setting. The format is a +list of dictionaries. The keys C, C, C, and C +are required. Optionally the C key can be used instead of the +C. For example: + + sshcollector: + - ip: '192.0.2.1' + user: oliver + password: letmein + platform: IOS + - hostname: 'core-router.example.com' + user: oliver + password: letmein + platform: IOS + +Platform is the final part of the classname to be instantiated to query the +host, e.g. platform B will be queried using +C. + +If the password is "-", public key authentication will be attempted. + +=back + +=head1 ADDING DEVICES + +Additional device classes can be easily integrated just by adding and +additonal class to the C namespace. +This class must implement an C method which returns +an array of hashrefs in the format + + @result = ({ ip => IPADDR, mac => MACADDR }, ...) + +The parameter C<$ssh> is an active C connection to the host. +Depending on the target system, it can be queried using simple methods like + + my @data = $ssh->capture("show whatever") + +or automated via Expect - this is mostly useful for non-Linux appliances which +don't support command execution via ssh: + + my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command"; + my $expect = Expect->init($pty); + my $prompt = qr/#/; + my ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt); + $expect->send("terminal length 0\n"); + # etc... + +The returned IP and MAC addresses should be in a format that the respective +B and B datatypes in PostgreSQL can handle. + +=head1 DEPENDENCIES + +=over 4 + +=item L + +=item L + +=back + +=head1 COPYRIGHT AND LICENSE + + Copyright (C) 2013 by the Netdisco Project + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Netdisco Project nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE NETDISCO DEVELOPER TEAM BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Initial Version for Netdisco 1.x + Copyright (C) 2013 by Christian Ramseyer (ramseyer@netnea.com) + I hereby grant full ownership of the code to the Netdisco Project + +=cut + +use warnings; +use strict; + +our $VERSION = 2.001000; +our $home; + +BEGIN { + use FindBin; + FindBin::again(); + + $home = ($ENV{NETDISCO_HOME} || $ENV{HOME}); + + # try to find a localenv if one isn't already in place. + if (!exists $ENV{PERL_LOCAL_LIB_ROOT}) { + use File::Spec; + my $localenv = File::Spec->catfile($FindBin::RealBin, 'localenv'); + exec($localenv, $0, @ARGV) if -f $localenv; + $localenv = File::Spec->catfile($home, 'perl5', 'bin', 'localenv'); + exec($localenv, $0, @ARGV) if -f $localenv; + + die "Sorry, can't find libs required for App::Netdisco.\n" + if !exists $ENV{PERLBREW_PERL}; + } +} + +BEGIN { + use Path::Class; + + # stuff useful locations into @INC and $PATH + unshift @INC, + dir($FindBin::RealBin)->parent->subdir('lib')->stringify, + dir($FindBin::RealBin, 'lib')->stringify; + + unshift @INC, + split m/:/, ($ENV{NETDISCO_INC} || ''); + + use Config; + $ENV{PATH} = $FindBin::RealBin . $Config{path_sep} . $ENV{PATH}; +} + +use App::Netdisco; +use App::Netdisco::Core::Arpnip 'store_arp'; +use App::Netdisco::Util::Node 'check_mac'; +use App::Netdisco::Util::DNS 'hostnames_resolve_async'; +use Dancer ':script'; + +use Data::Printer; +use Module::Load (); +use Net::OpenSSH; +use MCE::Loop Sereal => 1; + +#this may be helpful with SSH issues: +#$Net::OpenSSH::debug = ~0; + +MCE::Loop::init { chunk_size => 1 }; +my %stats; + +exit main(); + +sub main { + my @input = @{ setting('sshcollector') }; + + my @mce_result = mce_loop { + my ($mce, $chunk_ref, $chunk_id) = @_; + my $host = $chunk_ref->[0]; + + my $hostlabel = (!defined $host->{hostname} or $host->{hostname} eq "-") + ? $host->{ip} : $host->{hostname}; + + if ($hostlabel) { + my $ssh = Net::OpenSSH->new( + $hostlabel, + user => $host->{user}, + password => $host->{password}, + timeout => 30, + async => 0, + master_opts => [ + -o => "StrictHostKeyChecking=no", + -o => "BatchMode=no" + ], + ); + + MCE->gather( process($hostlabel, $ssh, $host) ); + } + } \@input; + + return 0 unless scalar @mce_result; + + foreach my $host (@mce_result) { + $stats{host}++; + info sprintf ' [%s] arpnip - retrieved %s entries', + $host->[0], scalar @{$host->[1]}; + store_arpentries($host->[1]); + } + + info sprintf 'arpnip - processed %s ARP Cache entries from %s devices', + $stats{entry}, $stats{host}; + return 0; +} + +sub process { + my ($hostlabel, $ssh, $args) = @_; + + my $class = "App::Netdisco::SSHCollector::Platform::".$args->{platform}; + Module::Load::load $class; + + my $device = $class->new(); + my $arpentries = [ $device->arpnip($hostlabel, $ssh, $args) ]; + + # debug p $arpentries; + if (scalar @$arpentries) { + hostnames_resolve_async($arpentries); + return [$hostlabel, $arpentries]; + } + else { + warning "WARNING: no entries received from <$hostlabel>"; + } +} + +sub store_arpentries { + my ($arpentries) = @_; + + foreach my $arpentry ( @$arpentries ) { + # skip broadcast/vrrp/hsrp and other wierdos + next unless check_mac( undef, $arpentry->{mac} ); + + debug sprintf ' arpnip - stored entry: %s / %s', + $arpentry->{mac}, $arpentry->{ip}; + store_arp({ + node => $arpentry->{mac}, + ip => $arpentry->{ip}, + dns => $arpentry->{dns}, + }); + + $stats{entry}++; + } +} + +__END__ + diff --git a/Netdisco/lib/App/Netdisco/SSHCollector/Platform/ACE.pm b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/ACE.pm new file mode 100644 index 00000000..9cb1b1b8 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/ACE.pm @@ -0,0 +1,87 @@ +package App::Netdisco::SSHCollector::Platform::ACE; + +# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4: + +=head1 NAME + +App::Netdisco::SSHCollector::Platform::ACE + +=head1 DESCRIPTION + +Collect ARP entries from Cisco ACE load balancers. ACEs have multiple +virtual contexts with individual ARP tables. Contexts are enumerated +with C, afterwards the commands C and +C must be executed for every context. + +The IOS shell does not permit to combine mulitple commands in a single +line, and Net::OpenSSH uses individual connections for individual commands, +so we need to use Expect to execute the changeto and show commands in +the same context. + +=cut + +use strict; +use warnings; + +use Moo; +use Expect; + +=head1 PUBLIC METHODS + +=over 4 + +=item B + +Retrieve ARP entries from device. C<$host> is the hostname or IP address +of the device. C<$ssh> is a Net::OpenSSH connection to the device. + +Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }. + +=cut + +sub arpnip{ + my ($self, $hostlabel, $ssh, @args) = @_; + + debug "$hostlabel $$ arpnip()"; + + my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command"; + my $expect = Expect->init($pty); + + my ($pos, $error, $match, $before, $after); + my $prompt = qr/#/; + + ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt); + + $expect->send("terminal length 0\n"); + ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); + + $expect->send("show context | include Name\n"); + ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); + + my @ctx; + my @arpentries; + + for (split(/\n/, $before)){ + if (m/Name: (\S+)/){ + push(@ctx, $1); + $expect->send("changeto $1\n"); + ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); + $expect->send("show arp\n"); + ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt); + for (split(/\n/, $before)){ + my ($ip, $mac) = split(/\s+/); + if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/[0-9a-f.]+/i) { + push(@arpentries, { ip => $ip, mac => $mac }); + } + } + + } + } + + $expect->send("exit\n"); + $expect->soft_close(); + + return @arpentries; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/SSHCollector/Platform/BigIP.pm b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/BigIP.pm new file mode 100644 index 00000000..7feb553b --- /dev/null +++ b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/BigIP.pm @@ -0,0 +1,66 @@ +package App::Netdisco::SSHCollector::Platform::BigIP; + +# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4: + +=head1 NAME + +NApp::etdisco::SSHCollector::Platform::BigIP + +=head1 DESCRIPTION + +Collect ARP entries from F5 BigIP load balancers. These are Linux boxes, +but feature an additional, proprietary IP stack which does not show +up in the standard SNMP ipNetToMediaTable. + +These devices also feature a CLI interface similar to IOS, which can +either be set as the login shell of the user, or be called from an +ordinary shell. This module assumes the former, and if "show net arp" +can't be executed, falls back to the latter. + +=cut + +use strict; +use warnings; + +use Moo; + +=head1 PUBLIC METHODS + +=over 4 + +=item B + +Retrieve ARP entries from device. C<$host> is the hostname or IP address +of the device. C<$ssh> is a Net::OpenSSH connection to the device. + +Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }. + +=cut +sub arpnip { + my ($self, $hostlabel, $ssh, @args) = @_; + + debug "$hostlabel $$ arpnip()"; + + my @data = $ssh->capture("show net arp"); + unless (@data){ + @data = $ssh->capture('tmsh -c "show net arp"'); + } + + chomp @data; + my @arpentries; + + foreach (@data){ + if (m/\d{1,3}\..*resolved/){ + my (undef, $ip, $mac) = split(/\s+/); + + # ips can look like 172.19.254.143%10, clean + $ip =~ s/%\d+//; + + push(@arpentries, {mac => $mac, ip => $ip}); + } + } + + return @arpentries; +} + +1; diff --git a/Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOS.pm b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOS.pm new file mode 100644 index 00000000..1165e6d5 --- /dev/null +++ b/Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOS.pm @@ -0,0 +1,55 @@ +package App::Netdisco::SSHCollector::Platform::IOS; + +# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4: + +=head1 NAME + +App::Netdisco::SSHCollector::Platform::IOS + +=head1 DESCRIPTION + +Collect ARP entries from Cisco IOS devices. + +=cut + +use strict; +use warnings; + +use Dancer ':script'; +use Data::Printer; +use Moo; + +=head1 PUBLIC METHODS + +=over 4 + +=item B + +Retrieve ARP entries from device. C<$host> is the hostname or IP address +of the device. C<$ssh> is a Net::OpenSSH connection to the device. + +Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }. + +=cut + +sub arpnip { + my ($self, $hostlabel, $ssh, @args) = @_; + + debug "$hostlabel $$ arpnip()"; + my @data = $ssh->capture("show ip arp"); + + chomp @data; + my @arpentries; + + # Internet 172.16.20.15 13 0024.b269.867d ARPA FastEthernet0/0.1 + foreach my $line (@data) { + next unless $line =~ m/^Internet/; + my @fields = split m/\s+/, $line; + + push @arpentries, { mac => $fields[3], ip => $fields[1] }; + } + + return @arpentries; +} + +1;