netdisco/bin/netdisco-sshcollector

#!/usr/bin/env perl

use warnings;
use strict;

our $home;

BEGIN {
  use FindBin;
  FindBin::again();

  $home = ($ENV{NETDISCO_HOME} || $ENV{HOME});

  # try to find a localenv if one isn't already in place.
  if (!exists $ENV{PERL_LOCAL_LIB_ROOT}) {
      use File::Spec;
      my $localenv = File::Spec->catfile($FindBin::RealBin, 'localenv');
      exec($localenv, $0, @ARGV) if -f $localenv;
      $localenv = File::Spec->catfile($home, 'perl5', 'bin', 'localenv');
      exec($localenv, $0, @ARGV) if -f $localenv;

      die "Sorry, can't find libs required for App::Netdisco.\n"
        if !exists $ENV{PERLBREW_PERL};
  }
}

BEGIN {
  use Path::Class;

  # stuff useful locations into @INC and $PATH
  unshift @INC,
    dir($FindBin::RealBin)->parent->subdir('lib')->stringify,
    dir($FindBin::RealBin, 'lib')->stringify;

  unshift @INC,
    split m/:/, ($ENV{NETDISCO_INC} || '');

  use Config;
  $ENV{PATH} = $FindBin::RealBin . $Config{path_sep} . $ENV{PATH};
}

use App::Netdisco;
use App::Netdisco::Util::Node qw/check_mac store_arp/;
use App::Netdisco::Util::FastResolver 'hostnames_resolve_async';
use Dancer ':script';

use Data::Printer;
use Module::Load ();
use Net::OpenSSH;
use MCE::Loop Sereal => 1;
use Pod::Usage 'pod2usage';

use Getopt::Long;
Getopt::Long::Configure ("bundling");

my ($debug, $sqltrace, $device, $opensshdebug, $workers) = (undef, 0, undef, undef, "auto");
my $result = GetOptions(
  'debug|D' => \$debug,
  'sqltrace|Q' => \$sqltrace,
  'device|d=s' => \$device,
  'opensshdebug|O' => \$opensshdebug,
  'workers|w=i' => \$workers,
) or pod2usage(
  -msg => 'error: bad options',
  -verbose => 0,
  -exitval => 1,
);

my $CONFIG = config();
$CONFIG->{logger} = 'console';
$CONFIG->{log} = ($debug ? 'debug' : 'info');
$ENV{DBIC_TRACE} ||= $sqltrace;

# reconfigure logging to force console output
Dancer::Logger->init('console', $CONFIG);

# silent exit unless explicitly requested
exit(0) unless setting('use_legacy_sshcollector');

if ($opensshdebug){
    $Net::OpenSSH::debug = ~0;
}

MCE::Loop::init { chunk_size => 1, max_workers => $workers };
my %stats;
$stats{entry} = 0;

exit main();

sub main {
    my @input = @{ setting('sshcollector') };

    if ($device){
        @input = grep{ ($_->{hostname} && $_->{hostname} eq $device) 
            || ($_->{ip} && $_->{ip} eq $device) } @input; 
    }

    #one-line Fisher-Yates from https://www.perlmonks.org/index.pl?node=Array%20One-Liners
    my ($i,$j) = (0);
    @input[-$i,$j] = @input[$j,-$i] while $j = rand(@input - $i), ++$i < @input;

    my @mce_result = mce_loop {
        my ($mce, $chunk_ref, $chunk_id) = @_;
        my $host = $chunk_ref->[0];

        my $hostlabel = (!defined $host->{hostname} or $host->{hostname} eq "-")
            ? $host->{ip} : $host->{hostname};

        if ($hostlabel) {
            my $ssh = Net::OpenSSH->new(
                $hostlabel,
                user => $host->{user},
                password => $host->{password},
                timeout => 30,
                async => 0,
                default_stderr_file => '/dev/null',
                master_opts => [
                    -o => "StrictHostKeyChecking=no",
                    -o => "BatchMode=no"
                ],
            );

           
            if ($ssh->error){ 
                warning "WARNING: Couldn't connect to <$hostlabel> - " . $ssh->error;
            }else{
                MCE->gather( process($hostlabel, $ssh, $host) );
            }
        }
    } \@input;

    return 0 unless scalar @mce_result;

    foreach my $host (@mce_result) {
        $stats{host}++;
        info sprintf ' [%s] arpnip - retrieved %s entries',
            $host->[0], scalar @{$host->[1]};
        store_arpentries($host->[1]);
    }

    info sprintf 'arpnip - processed %s ARP Cache entries from %s devices',
        $stats{entry}, $stats{host};
    return 0;
}

sub process {
    my ($hostlabel, $ssh, $args) = @_;

    my $class = "App::Netdisco::SSHCollector::Platform::".$args->{platform};
    Module::Load::load $class;

    my $device = $class->new();
    my $arpentries = [ $device->arpnip($hostlabel, $ssh, $args) ];

    # debug p $arpentries;
    if (not scalar @$arpentries) {
        warning "WARNING: no entries received from <$hostlabel>";
    }
    hostnames_resolve_async($arpentries);
    return [$hostlabel, $arpentries];
}

sub store_arpentries {
    my ($arpentries) = @_;

    foreach my $arpentry ( @$arpentries ) {
        # skip broadcast/vrrp/hsrp and other wierdos
        next unless check_mac( $arpentry->{mac} );

        debug sprintf '  arpnip - stored entry: %s / %s',
            $arpentry->{mac}, $arpentry->{ip};
        store_arp({
            node => $arpentry->{mac},
            ip => $arpentry->{ip},
            dns => $arpentry->{dns},
        });

        $stats{entry}++;
    }
}

=head1 NAME

netdisco-sshcollector - Collect ARP data for Netdisco from devices without
full SNMP support

=head1 SYNOPSIS

 # install dependencies:
 ~/bin/localenv cpanm --notest Net::OpenSSH Expect

 # run manually, or add to cron:
 ~/bin/netdisco-sshcollector [-DQO] [-w <max_workers>] 

 # limit run to a single device defined in the config
 ~/bin/netdisco-sshcollector [-DQO] [-w <max_workers>] -d <device> 

=head1 DESCRIPTION

Collects ARP data for Netdisco from devices without full SNMP support.
Currently, ARP tables can be retrieved from the following device classes:

=over 4

=item * L<App::Netdisco::SSHCollector::Platform::GAIAEmbedded> - Check Point GAIA Embedded

=item * L<App::Netdisco::SSHCollector::Platform::CPVSX> - Check Point VSX

=item * L<App::Netdisco::SSHCollector::Platform::ACE> - Cisco ACE

=item * L<App::Netdisco::SSHCollector::Platform::ASA> - Cisco ASA

=item * L<App::Netdisco::SSHCollector::Platform::IOS> - Cisco IOS

=item * L<App::Netdisco::SSHCollector::Platform::IOSXR> - Cisco IOS XR

=item * L<App::Netdisco::SSHCollector::Platform::NXOS> - Cisco NXOS

=item * L<App::Netdisco::SSHCollector::Platform::BigIP> - F5 Networks BigIP

=item * L<App::Netdisco::SSHCollector::Platform::FreeBSD> - FreeBSD

=item * L<App::Netdisco::SSHCollector::Platform::Linux> - Linux

=item * L<App::Netdisco::SSHCollector::Platform::PaloAlto> - Palo Alto

=back

The collected arp entries are then directly stored in the netdisco database.

=head1 CONFIGURATION

The following should go into your Netdisco configuration file,
F<~/environments/deployment.yml>.

=over 4

=item C<sshcollector>

Data is collected from the machines specified in this setting. The format is a
list of dictionaries. The keys C<ip>, C<user>, C<password>, and C<platform>
are required. Optionally the C<hostname> key can be used instead of the
C<ip>. For example:

 sshcollector:
   - ip: '192.0.2.1'
     user: oliver
     password: letmein
     platform: IOS
   - hostname: 'core-router.example.com'
     user: oliver
     password:
     platform: IOS

Platform is the final part of the classname to be instantiated to query the
host, e.g. platform B<ACE> will be queried using
C<App::Netdisco::SSHCollector::Platform::ACE>.

If the password is blank, public key authentication will be attempted with the
default key for the netdisco user. Password protected keys are currently not
supported.

=back

=head1 ADDING DEVICES

Additional device classes can be easily integrated just by adding and
additonal class to the C<App::Netdisco::SSHCollector::Platform> namespace.
This class must implement an C<arpnip($hostname, $ssh)> method which returns
an array of hashrefs in the format

 @result = ({ ip => IPADDR, mac => MACADDR }, ...) 

The parameter C<$ssh> is an active C<Net::OpenSSH> connection to the host.
Depending on the target system, it can be queried using simple methods like

 my @data = $ssh->capture("show whatever")

or automated via Expect - this is mostly useful for non-Linux appliances which
don't support command execution via ssh:

 my ($pty, $pid) = $ssh->open2pty;
 unless ($pty) {
   debug "unable to run remote command [$hostlabel] " . $ssh->error;
   return ();
 }
 my $expect = Expect->init($pty);
 my $prompt = qr/#/;
 my ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
 $expect->send("terminal length 0\n");
 # etc...

The returned IP and MAC addresses should be in a format that the respective
B<inetaddr> and B<macaddr> datatypes in PostgreSQL can handle.   

=head1 COMMAND LINE OPTIONS

=over 4

=item C<-D>

Netdisco debug log level.

=item C<-Q>

L<DBIx::Class> trace enabled.

=item C<-O>

L<Net::OpenSSH> trace enabled.

=item C<-w>

Set maximum parallel workers for L<MCE::Loop>. The default is B<auto>. 

=item C<-d device>

Only run for a single device. Takes an IP or hostname, must exactly match the
value in the config file.

=back

=head1 DEPENDENCIES

=over 4

=item L<App::Netdisco>

=item L<Net::OpenSSH>

=item L<Expect>

=item L<http://www.openssh.com/>

=back

=cut