netdisco-sshcollector script to get ARP data on devices without SNMP

Netdisco/Changes

@@ -1,3 +1,9 @@
+2.029006 - 
+
+  [NEW FEATURES]
+
+  * netdisco-sshcollector script to get ARP data on devices without SNMP (C. Ramseyer)
+
 2.029005 - 2014-08-13
 
   [ENHANCEMENTS]

Netdisco/MANIFEST

@@ -6,6 +6,7 @@ bin/netdisco-db-deploy
 bin/netdisco-deploy
 bin/netdisco-do
 bin/netdisco-rancid-export
+bin/netdisco-sshcollector
 bin/netdisco-web
 bin/netdisco-web-fg
 Changes
@@ -148,6 +149,9 @@ lib/App/Netdisco/Manual/Deployment.pod
 lib/App/Netdisco/Manual/Developing.pod
 lib/App/Netdisco/Manual/ReleaseNotes.pod
 lib/App/Netdisco/Manual/WritingPlugins.pod
+lib/App/Netdisco/SSHCollector/Platform/ACE.pm
+lib/App/Netdisco/SSHCollector/Platform/BigIP.pm
+lib/App/Netdisco/SSHCollector/Platform/IOS.pm
 lib/App/Netdisco/Util/Daemon.pm
 lib/App/Netdisco/Util/Device.pm
 lib/App/Netdisco/Util/DNS.pm

Netdisco/META.yml

@@ -24,6 +24,7 @@ no_index:
 recommends:
   Graph: 0
   GraphViz: 0
+  Net::OpenSSH: 0
 requires:
   Algorithm::Cron: 0.07
   AnyEvent: 7.05
@@ -69,7 +70,6 @@ requires:
   Sereal: 0
   Socket6: 0.23
   Starman: 0.4008
-  Sys::Proctitle: 0
   Template: 2.24
   Template::Plugin::CSV: 0.04
   Template::Plugin::Number::Format: 1.02

Netdisco/Makefile.PL

@@ -70,6 +70,7 @@ if ( $^O eq 'linux' ) {
 
 recommends 'Graph' => 0;
 recommends 'GraphViz' => 0;
+recommends 'Net::OpenSSH' => 0;
 
 install_share 'share';
 
@@ -82,6 +83,7 @@ install_script 'bin/netdisco-daemon';
 install_script 'bin/netdisco-web-fg';
 install_script 'bin/netdisco-web';
 install_script 'bin/netdisco-rancid-export';
+install_script 'bin/netdisco-sshcollector';
 
 resources
   homepage => 'http://netdisco.org/',

Netdisco/bin/netdisco-sshcollector

@@ -0,0 +1,268 @@
+#!/usr/bin/env perl
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+netdisco-sshcollector - Collect ARP data for Netdisco from devices without
+full SNMP support
+
+=head1 DESCRIPTION
+
+Collects ARP data for Netdisco from devices without full SNMP support.
+Currently, ARP tables can be retrieved from the following device classes:
+
+=over 4
+
+=item * L<App::Netdisco::SSHCollector::Platform::ACE> - Cisco ACE (Application Control Engine) 
+
+=item * L<App::Netdisco::SSHCollector::Platform::BigIP> - F5 Networks BigIP
+
+=item * L<App::Netdisco::SSHCollector::Platform::IOS> - Cisco IOS
+
+=back
+
+The collected arp entries are then directly stored in the netdisco database.
+
+=head1 CONFIGURATION
+
+The following should go into your Netdisco 2 configuration file, "C<<
+~/environments/deployment.yml >>"
+
+=over 4
+
+=item C<sshcollector>
+
+Data is collected from the machines specified in this setting. The format is a
+list of dictionaries. The keys C<ip>, C<user>, C<password>, and C<platform>
+are required.  Optionally the C<hostname> key can be used instead of the
+C<ip>. For example:
+
+ sshcollector:
+   - ip: '192.0.2.1'
+     user: oliver
+     password: letmein
+     platform: IOS
+   - hostname: 'core-router.example.com'
+     user: oliver
+     password: letmein
+     platform: IOS
+
+Platform is the final part of the classname to be instantiated to query the
+host, e.g. platform B<ACE> will be queried using
+C<App::Netdisco::SSHCollector::Platform::ACE>.
+
+If the password is "-", public key authentication will be attempted.
+
+=back
+
+=head1 ADDING DEVICES
+
+Additional device classes can be easily integrated just by adding and
+additonal class to the C<App::Netdisco::SSHCollector::Platform> namespace.
+This class must implement an C<arpnip($hostname, $ssh)> method which returns
+an array of hashrefs in the format
+
+ @result = ({ ip => IPADDR, mac => MACADDR }, ...) 
+
+The parameter C<$ssh> is an active C<Net::OpenSSH> connection to the host.
+Depending on the target system, it can be queried using simple methods like
+
+ my @data = $ssh->capture("show whatever")
+
+or automated via Expect - this is mostly useful for non-Linux appliances which
+don't support command execution via ssh:
+
+ my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
+ my $expect = Expect->init($pty);
+ my $prompt = qr/#/;
+ my ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+ $expect->send("terminal length 0\n");
+ # etc...
+
+The returned IP and MAC addresses should be in a format that the respective
+B<inetaddr> and B<macaddr> datatypes in PostgreSQL can handle.   
+
+=head1 DEPENDENCIES
+
+=over 4
+
+=item L<App::Netdisco>
+
+=item L<Net::OpenSSH>
+
+=back
+
+=head1 COPYRIGHT AND LICENSE
+
+ Copyright (C) 2013 by the Netdisco Project
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+     * Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+     * Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+     * Neither the name of the Netdisco Project nor the
+       names of its contributors may be used to endorse or promote products
+       derived from this software without specific prior written permission.
+ 
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE NETDISCO DEVELOPER TEAM BE LIABLE FOR ANY
+ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ Initial Version for Netdisco 1.x
+ Copyright (C) 2013 by Christian Ramseyer (ramseyer@netnea.com)
+ I hereby grant full ownership of the code to the Netdisco Project
+ 
+=cut
+
+use warnings;
+use strict;
+
+our $VERSION = 2.001000;
+our $home;
+
+BEGIN {
+  use FindBin;
+  FindBin::again();
+
+  $home = ($ENV{NETDISCO_HOME} || $ENV{HOME});
+
+  # try to find a localenv if one isn't already in place.
+  if (!exists $ENV{PERL_LOCAL_LIB_ROOT}) {
+      use File::Spec;
+      my $localenv = File::Spec->catfile($FindBin::RealBin, 'localenv');
+      exec($localenv, $0, @ARGV) if -f $localenv;
+      $localenv = File::Spec->catfile($home, 'perl5', 'bin', 'localenv');
+      exec($localenv, $0, @ARGV) if -f $localenv;
+
+      die "Sorry, can't find libs required for App::Netdisco.\n"
+        if !exists $ENV{PERLBREW_PERL};
+  }
+}
+
+BEGIN {
+  use Path::Class;
+
+  # stuff useful locations into @INC and $PATH
+  unshift @INC,
+    dir($FindBin::RealBin)->parent->subdir('lib')->stringify,
+    dir($FindBin::RealBin, 'lib')->stringify;
+
+  unshift @INC,
+    split m/:/, ($ENV{NETDISCO_INC} || '');
+
+  use Config;
+  $ENV{PATH} = $FindBin::RealBin . $Config{path_sep} . $ENV{PATH};
+}
+
+use App::Netdisco;
+use App::Netdisco::Core::Arpnip 'store_arp';
+use App::Netdisco::Util::Node 'check_mac';
+use App::Netdisco::Util::DNS 'hostnames_resolve_async';
+use Dancer ':script';
+
+use Data::Printer;
+use Module::Load ();
+use Net::OpenSSH;
+use MCE::Loop Sereal => 1;
+
+#this may be helpful with SSH issues:
+#$Net::OpenSSH::debug = ~0;
+
+MCE::Loop::init { chunk_size => 1 };
+my %stats;
+
+exit main();
+
+sub main {
+    my @input = @{ setting('sshcollector') };
+
+    my @mce_result = mce_loop {
+        my ($mce, $chunk_ref, $chunk_id) = @_;
+        my $host = $chunk_ref->[0];
+
+        my $hostlabel = (!defined $host->{hostname} or $host->{hostname} eq "-")
+            ? $host->{ip} : $host->{hostname};
+
+        if ($hostlabel) {
+            my $ssh = Net::OpenSSH->new(
+                $hostlabel,
+                user => $host->{user},
+                password => $host->{password},
+                timeout => 30,
+                async => 0,
+                master_opts => [
+                    -o => "StrictHostKeyChecking=no",
+                    -o => "BatchMode=no"
+                ],
+            );
+
+            MCE->gather( process($hostlabel, $ssh, $host) );
+        }
+    } \@input;
+
+    return 0 unless scalar @mce_result;
+
+    foreach my $host (@mce_result) {
+        $stats{host}++;
+        info sprintf ' [%s] arpnip - retrieved %s entries',
+            $host->[0], scalar @{$host->[1]};
+        store_arpentries($host->[1]);
+    }
+
+    info sprintf 'arpnip - processed %s ARP Cache entries from %s devices',
+        $stats{entry}, $stats{host};
+    return 0;
+}
+
+sub process {
+    my ($hostlabel, $ssh, $args) = @_;
+
+    my $class = "App::Netdisco::SSHCollector::Platform::".$args->{platform};
+    Module::Load::load $class;
+
+    my $device = $class->new();
+    my $arpentries = [ $device->arpnip($hostlabel, $ssh, $args) ];
+
+    # debug p $arpentries;
+    if (scalar @$arpentries) {
+        hostnames_resolve_async($arpentries);
+        return [$hostlabel, $arpentries];
+    }
+    else {
+        warning "WARNING: no entries received from <$hostlabel>";
+    }
+}
+
+sub store_arpentries {
+    my ($arpentries) = @_;
+
+    foreach my $arpentry ( @$arpentries ) {
+        # skip broadcast/vrrp/hsrp and other wierdos
+        next unless check_mac( undef, $arpentry->{mac} );
+
+        debug sprintf '  arpnip - stored entry: %s / %s',
+            $arpentry->{mac}, $arpentry->{ip};
+        store_arp({
+            node => $arpentry->{mac},
+            ip => $arpentry->{ip},
+            dns => $arpentry->{dns},
+        });
+
+        $stats{entry}++;
+    }
+}
+
+__END__
+

Netdisco/lib/App/Netdisco/SSHCollector/Platform/ACE.pm

@@ -0,0 +1,87 @@
+package App::Netdisco::SSHCollector::Platform::ACE;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::ACE
+
+=head1 DESCRIPTION
+
+Collect ARP entries from Cisco ACE load balancers. ACEs have multiple
+virtual contexts with individual ARP tables. Contexts are enumerated
+with C<show context>, afterwards the commands C<changeto CONTEXTNAME> and
+C<show arp> must be executed for every context.
+
+The IOS shell does not permit to combine mulitple commands in a single
+line, and Net::OpenSSH uses individual connections for individual commands,
+so we need to use Expect to execute the changeto and show commands in
+the same context.
+
+=cut
+
+use strict;
+use warnings;
+
+use Moo;
+use Expect;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip{
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
+    my $expect = Expect->init($pty);
+
+    my ($pos, $error, $match, $before, $after);
+    my $prompt = qr/#/;
+
+    ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+
+    $expect->send("terminal length 0\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    $expect->send("show context | include Name\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    my @ctx;
+    my @arpentries;
+
+    for (split(/\n/, $before)){
+        if (m/Name: (\S+)/){
+            push(@ctx, $1);
+            $expect->send("changeto $1\n");
+            ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+            $expect->send("show arp\n");
+            ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+            for (split(/\n/, $before)){
+                my ($ip, $mac) = split(/\s+/);
+                if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/[0-9a-f.]+/i) {
+                    push(@arpentries, { ip => $ip, mac => $mac });
+                }
+            }
+
+        }
+    }
+
+    $expect->send("exit\n");
+    $expect->soft_close();
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/BigIP.pm

@@ -0,0 +1,66 @@
+package App::Netdisco::SSHCollector::Platform::BigIP;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+NApp::etdisco::SSHCollector::Platform::BigIP
+
+=head1 DESCRIPTION
+
+Collect ARP entries from F5 BigIP load balancers. These are Linux boxes,
+but feature an additional, proprietary IP stack which does not show
+up in the standard SNMP ipNetToMediaTable.
+
+These devices also feature a CLI interface similar to IOS, which can
+either be set as the login shell of the user, or be called from an
+ordinary shell. This module assumes the former, and if "show net arp"
+can't be executed, falls back to the latter.
+
+=cut
+
+use strict;
+use warnings;
+
+use Moo;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+sub arpnip {
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my @data = $ssh->capture("show net arp");
+    unless (@data){
+        @data = $ssh->capture('tmsh -c "show net arp"');
+    }
+
+    chomp @data;
+    my @arpentries;
+
+    foreach (@data){
+        if (m/\d{1,3}\..*resolved/){
+            my (undef, $ip, $mac) = split(/\s+/);
+
+            # ips can look like 172.19.254.143%10, clean
+            $ip =~ s/%\d+//;
+
+            push(@arpentries, {mac => $mac, ip => $ip});
+        }
+    }
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOS.pm

@@ -0,0 +1,55 @@
+package App::Netdisco::SSHCollector::Platform::IOS;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::IOS
+
+=head1 DESCRIPTION
+
+Collect ARP entries from Cisco IOS devices.
+
+=cut
+
+use strict;
+use warnings;
+
+use Dancer ':script';
+use Data::Printer;
+use Moo;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip {
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+    my @data = $ssh->capture("show ip arp");
+
+    chomp @data;
+    my @arpentries;
+
+    # Internet  172.16.20.15   13   0024.b269.867d  ARPA FastEthernet0/0.1
+    foreach my $line (@data) {
+        next unless $line =~ m/^Internet/;
+        my @fields = split m/\s+/, $line;
+
+        push @arpentries, { mac => $fields[3], ip => $fields[1] };
+    }
+
+    return @arpentries;
+}
+
+1;