Merge branch 'master' into em-device-ports-json

Conflicts:
	Netdisco/share/public/css/netdisco.css
	Netdisco/share/views/ajax/device/ports.tt

Netdisco/lib/App/Netdisco.pm

@@ -4,7 +4,7 @@ use strict;
 use warnings;
 use 5.010_000;
 
-our $VERSION = '2.029005';
+our $VERSION = '2.029007';
 use App::Netdisco::Configuration;
 
 use Module::Find ();

Netdisco/lib/App/Netdisco/Core/Arpnip.pm

@@ -96,14 +96,21 @@ sub _get_arps {
   return $resolved_ips;
 }
 
-=head2 store_arp( $mac, $ip, $name, $now? )
+=head2 store_arp( \%host, $now? )
 
 Stores a new entry to the C<node_ip> table with the given MAC, IP (v4 or v6)
-and DNS host name.
+and DNS host name. Host details are provided in a Hash ref:
 
-Will mark old entries for this IP as no longer C<active>.
+ {
+    ip   => '192.0.2.1',
+    node => '00:11:22:33:44:55',
+    dns  => 'myhost.example.com',
+ }
 
-Optionally a literal string can be passed in the fourth argument for the
+The C<dns> entry is optional. The update will mark old entries for this IP as
+no longer C<active>.
+
+Optionally a literal string can be passed in the second argument for the
 C<time_last> timestamp, otherwise the current timestamp (C<now()>) is used.
 
 =cut

Netdisco/lib/App/Netdisco/Core/Discover.pm

@@ -120,7 +120,8 @@ sub store_device {
   # build device aliases suitable for DBIC
   my @aliases;
   foreach my $entry (keys %$ip_index) {
-      my $ip = NetAddr::IP::Lite->new($entry);
+      my $ip = NetAddr::IP::Lite->new($entry)
+        or next;
       my $addr = $ip->addr;
 
       next if $addr eq '0.0.0.0';

Netdisco/lib/App/Netdisco/Core/Macsuck.pm

@@ -346,6 +346,8 @@ sub _walk_fwtable {
           next;
       }
 
+      my $vlan = $fw_vlan->{$idx} || $comm_vlan || '0';
+
       # check to see if the port is connected to another device
       # and if we have that device in the database.
 
@@ -371,12 +373,13 @@ sub _walk_fwtable {
               debug sprintf
                 ' [%s] macsuck %s - port %s is detected uplink - skipping.',
                 $device->ip, $mac, $port;
+
+              $skiplist->{$port} = [ $vlan, $mac ] # remember for later
+                if exists $port_macs->{$mac};
               next;
           }
       }
 
-      my $vlan = $fw_vlan->{$idx} || $comm_vlan || '0';
-
       if (exists $port_macs->{$mac}) {
           my $switch_ip = $port_macs->{$mac};
           if ($device->ip eq $switch_ip) {
@@ -395,7 +398,8 @@ sub _walk_fwtable {
           if (not setting('macsuck_bleed')) {
                 debug sprintf ' [%s] macsuck %s - adding port %s to skiplist',
                     $device->ip, $mac, $port;
-                $skiplist->{$port} = delete $cache->{$vlan}->{$port};
+
+                $skiplist->{$port} = [ $vlan, $mac ]; # remember for later
                 next;
           }
       }
@@ -410,6 +414,15 @@ sub _walk_fwtable {
       ++$cache->{$vlan}->{$port}->{$mac};
   }
 
+  # restore MACs of neighbor devices.
+  # this is when we have a "possible uplink" detected but we still want to
+  # record the single MAC of the neighbor device so it works in Node search.
+  foreach my $port (keys %$skiplist) {
+      my ($vlan, $mac) = @{ $skiplist->{$port} };
+      delete $cache->{$_}->{$port} for keys %$cache; # nuke nodes on all VLANs
+      ++$cache->{$vlan}->{$port}->{$mac};
+  }
+
   return $cache;
 }
 

Netdisco/lib/App/Netdisco/DB/Result/Virtual/NodesDiscovered.pm

@@ -1,4 +1,4 @@
-package App::Netdisco::DB::Result::Virtual::PhonesDiscovered;
+package App::Netdisco::DB::Result::Virtual::NodesDiscovered;
 
 use strict;
 use warnings;
@@ -7,7 +7,7 @@ use base 'DBIx::Class::Core';
 
 __PACKAGE__->table_class('DBIx::Class::ResultSource::View');
 
-__PACKAGE__->table('phones_discovered');
+__PACKAGE__->table('nodes_discovered');
 __PACKAGE__->result_source_instance->is_virtual(1);
 __PACKAGE__->result_source_instance->view_definition(<<ENDSQL
 SELECT d.ip,
@@ -33,9 +33,7 @@ WHERE d.ip = p.ip
      WHERE a.alias = p.remote_ip
        AND q.ip = a.ip
        AND q.port = p.remote_port)
-  AND p.remote_ip IS NOT NULL
-  AND p.remote_port IS NOT NULL
-  AND p.remote_type ILIKE '%ip_phone%'
+  AND (p.remote_id IS NOT NULL OR p.remote_type IS NOT NULL)
 ENDSQL
 );
 

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -312,7 +312,20 @@ Set to "C<false>" if you MUST maintain backwards compatibility with the Netdisco
 
 Value: Number. Default: 10.
 
-The number of rows in a table page.
+The default number of rows in a table page.
+
+=head3 C<table_showrecordsmenu>
+
+Value: Number. Default:
+
+ table_showrecordsmenu:
+   - [10, 25, 50, 100, '-1']
+   - [10, 25, 50, 100, 'All']
+
+The choices available to users for selecting the number of rows per page. The
+format is two lists: one of the values and one of the labels in the web
+interface. You can see in the default that a value of "C<-1>" means Show All
+Records.
 
 =head2 Netdisco Core
 

Netdisco/lib/App/Netdisco/Manual/Deployment.pod

@@ -164,7 +164,7 @@ Then run the web daemon with the environment variable to enable the feature:
 
  DANCER_DEBUG=1 ~/bin/netdisco-web restart
 
-=head2 Database Backups
+=head1 Database Backups
 
 We recommend you backup the Netdisco database regularly. You could put the
 following commands into a shell script and call it nightly from C<cron>:

Netdisco/lib/App/Netdisco/SSHCollector/Platform/ACE.pm

@@ -0,0 +1,87 @@
+package App::Netdisco::SSHCollector::Platform::ACE;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::ACE
+
+=head1 DESCRIPTION
+
+Collect ARP entries from Cisco ACE load balancers. ACEs have multiple
+virtual contexts with individual ARP tables. Contexts are enumerated
+with C<show context>, afterwards the commands C<changeto CONTEXTNAME> and
+C<show arp> must be executed for every context.
+
+The IOS shell does not permit to combine mulitple commands in a single
+line, and Net::OpenSSH uses individual connections for individual commands,
+so we need to use Expect to execute the changeto and show commands in
+the same context.
+
+=cut
+
+use strict;
+use warnings;
+
+use Moo;
+use Expect;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip{
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
+    my $expect = Expect->init($pty);
+
+    my ($pos, $error, $match, $before, $after);
+    my $prompt = qr/#/;
+
+    ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+
+    $expect->send("terminal length 0\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    $expect->send("show context | include Name\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    my @ctx;
+    my @arpentries;
+
+    for (split(/\n/, $before)){
+        if (m/Name: (\S+)/){
+            push(@ctx, $1);
+            $expect->send("changeto $1\n");
+            ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+            $expect->send("show arp\n");
+            ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+            for (split(/\n/, $before)){
+                my ($ip, $mac) = split(/\s+/);
+                if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/[0-9a-f.]+/i) {
+                    push(@arpentries, { ip => $ip, mac => $mac });
+                }
+            }
+
+        }
+    }
+
+    $expect->send("exit\n");
+    $expect->soft_close();
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/BigIP.pm

@@ -0,0 +1,66 @@
+package App::Netdisco::SSHCollector::Platform::BigIP;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+NApp::etdisco::SSHCollector::Platform::BigIP
+
+=head1 DESCRIPTION
+
+Collect ARP entries from F5 BigIP load balancers. These are Linux boxes,
+but feature an additional, proprietary IP stack which does not show
+up in the standard SNMP ipNetToMediaTable.
+
+These devices also feature a CLI interface similar to IOS, which can
+either be set as the login shell of the user, or be called from an
+ordinary shell. This module assumes the former, and if "show net arp"
+can't be executed, falls back to the latter.
+
+=cut
+
+use strict;
+use warnings;
+
+use Moo;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+sub arpnip {
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my @data = $ssh->capture("show net arp");
+    unless (@data){
+        @data = $ssh->capture('tmsh -c "show net arp"');
+    }
+
+    chomp @data;
+    my @arpentries;
+
+    foreach (@data){
+        if (m/\d{1,3}\..*resolved/){
+            my (undef, $ip, $mac) = split(/\s+/);
+
+            # ips can look like 172.19.254.143%10, clean
+            $ip =~ s/%\d+//;
+
+            push(@arpentries, {mac => $mac, ip => $ip});
+        }
+    }
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOS.pm

@@ -0,0 +1,55 @@
+package App::Netdisco::SSHCollector::Platform::IOS;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::IOS
+
+=head1 DESCRIPTION
+
+Collect ARP entries from Cisco IOS devices.
+
+=cut
+
+use strict;
+use warnings;
+
+use Dancer ':script';
+use Data::Printer;
+use Moo;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip {
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+    my @data = $ssh->capture("show ip arp");
+
+    chomp @data;
+    my @arpentries;
+
+    # Internet  172.16.20.15   13   0024.b269.867d  ARPA FastEthernet0/0.1
+    foreach my $line (@data) {
+        next unless $line =~ m/^Internet/;
+        my @fields = split m/\s+/, $line;
+
+        push @arpentries, { mac => $fields[3], ip => $fields[1] };
+    }
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm

@@ -0,0 +1,68 @@
+package App::Netdisco::SSHCollector::Platform::IOSXR;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::IOSXR
+
+=head1 DESCRIPTION
+
+Collect ARP entries from Cisco IOS XR devices.
+
+=cut
+
+use strict;
+use warnings;
+
+use Dancer ':script';
+use Data::Printer;
+use Moo;
+use Expect;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip {
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
+    my $expect = Expect->init($pty);
+
+    my ($pos, $error, $match, $before, $after);
+    my $prompt = qr/#/;
+
+    ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+
+    $expect->send("terminal length 0\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    my @arpentries;
+
+    $expect->send("show arp vrf all\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
+
+    # 0.0.0.0     00:00:00   0000.0000.0000  Dynamic    ARPA  GigabitEthernet0/0/0/0
+    for (split(/\n/, $before)){
+        my ($ip, $age, $mac, $state, $t, $iface) = split(/\s+/);
+        if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/[0-9a-f.]+/i) {
+            push(@arpentries, { ip => $ip, mac => $mac });
+        }
+    }
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm

@@ -0,0 +1,66 @@
+package App::Netdisco::SSHCollector::Platform::PaloAlto;
+
+# vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4:
+
+=head1 NAME
+
+App::Netdisco::SSHCollector::Platform::PaloAlto
+
+=head1 DESCRIPTION
+
+Collect ARP entries from PaloAlto devices.
+
+=cut
+
+use strict;
+use warnings;
+
+use Dancer ':script';
+use Data::Printer;
+use Moo;
+use Expect;
+
+=head1 PUBLIC METHODS
+
+=over 4
+
+=item B<arpnip($host, $ssh)>
+
+Retrieve ARP entries from device. C<$host> is the hostname or IP address
+of the device. C<$ssh> is a Net::OpenSSH connection to the device.
+
+Returns an array of hashrefs in the format { mac => MACADDR, ip => IPADDR }.
+
+=cut
+
+sub arpnip{
+    my ($self, $hostlabel, $ssh, @args) = @_;
+
+    debug "$hostlabel $$ arpnip()";
+
+    my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
+    my $expect = Expect->init($pty);
+    my ($pos, $error, $match, $before, $after);
+    my $prompt = qr/> \r?$/;
+
+    ($pos, $error, $match, $before, $after) = $expect->expect(20, -re, $prompt);
+    $expect->send("set cli pager off\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+    $expect->send("show arp all\n");
+    ($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
+
+    my @arpentries;
+    for (split(/\r\n/, $before)){
+        next unless $_ =~ m/(\d{1,3}\.){3}\d{1,3}/;
+        my ($tmp, $ip, $mac) = split(/\s+/);
+        if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/([0-9a-f]{2}:){5}[0-9a-f]{2}/i) {
+             push(@arpentries, { ip => $ip, mac => $mac });
+        }
+    }
+    $expect->send("exit\n");
+    $expect->soft_close();
+
+    return @arpentries;
+}
+
+1;

Netdisco/lib/App/Netdisco/Util/DNS.pm

@@ -166,7 +166,8 @@ sub no_resolve {
     my $config = setting('dns')->{no} || [];
     return 0 if not scalar @$config;
 
-    my $addr = NetAddr::IP::Lite->new($ip);
+    my $addr = NetAddr::IP::Lite->new($ip)
+      or return 1;
 
     foreach my $item (@$config) {
         my $c_ip = NetAddr::IP::Lite->new($item)

Netdisco/lib/App/Netdisco/Web.pm

@@ -76,6 +76,10 @@ hook 'before_template' => sub {
     # create date ranges from within templates
     $tokens->{to_daterange}  = sub { interval_to_daterange(@_) };
 
+    # data structure for DataTables records per page menu
+    $tokens->{table_showrecordsmenu} =
+      to_json( setting('table_showrecordsmenu') );
+
     # fix Plugin Template Variables to be only path+query
     $tokens->{$_} = $tokens->{$_}->path_query
       for qw/search_node search_device device_ports/;

Netdisco/lib/App/Netdisco/Web/Plugin/Report/NodesDiscovered.pm

@@ -0,0 +1,54 @@
+package App::Netdisco::Web::Plugin::Report::NodesDiscovered;
+
+use Dancer ':syntax';
+use Dancer::Plugin::DBIC;
+use Dancer::Plugin::Auth::Extensible;
+
+use App::Netdisco::Web::Plugin;
+use App::Netdisco::Util::Web 'sql_match';
+
+register_report(
+    {   category     => 'Node',
+        tag          => 'nodesdiscovered',
+        label        => 'Nodes discovered through LLDP/CDP',
+        provides_csv => 1,
+    }
+);
+
+get '/ajax/content/report/nodesdiscovered' => require_login sub {
+    my $op = param('matchall') ? '-and' : '-or';
+
+    my @results = schema('netdisco')->resultset('Virtual::NodesDiscovered')
+        ->search({
+            $op => [
+              (param('aps') ?
+                ('me.remote_type' => { -ilike => 'AP:%' }) : ()),
+              (param('phones') ?
+                ('me.remote_type' => { -ilike => '%ip_phone%' }) : ()),
+              (param('remote_id') ?
+                ('me.remote_id' => { -ilike => scalar sql_match(param('remote_id')) }) : ()),
+              (param('remote_type') ? ('-or' => [
+                map  {( 'me.remote_type' => { -ilike => scalar sql_match($_) } )}
+                grep { $_ }
+                     (ref param('remote_type') ? @{param('remote_type')} : param('remote_type'))
+                ]) : ()),
+            ],
+        })
+        ->hri->all;
+
+    return unless scalar @results;
+
+    if ( request->is_ajax ) {
+        my $json = to_json( \@results );
+        template 'ajax/report/nodesdiscovered.tt', { results => $json },
+            { layout => undef };
+    }
+    else {
+        header( 'Content-Type' => 'text/comma-separated-values' );
+        template 'ajax/report/nodesdiscovered_csv.tt',
+            { results => \@results },
+            { layout  => undef };
+    }
+};
+
+1;

Netdisco/lib/App/Netdisco/Web/Plugin/Report/PhonesDiscovered.pm

@@ -1,36 +0,0 @@
-package App::Netdisco::Web::Plugin::Report::PhonesDiscovered;
-
-use Dancer ':syntax';
-use Dancer::Plugin::DBIC;
-use Dancer::Plugin::Auth::Extensible;
-
-use App::Netdisco::Web::Plugin;
-
-register_report(
-    {   category     => 'Node',
-        tag          => 'phonesdiscovered',
-        label        => 'IP Phones discovered through LLDP/CDP',
-        provides_csv => 1,
-    }
-);
-
-get '/ajax/content/report/phonesdiscovered' => require_login sub {
-    my @results = schema('netdisco')->resultset('Virtual::PhonesDiscovered')
-        ->hri->all;
-
-    return unless scalar @results;
-
-    if ( request->is_ajax ) {
-        my $json = to_json( \@results );
-        template 'ajax/report/phonesdiscovered.tt', { results => $json },
-            { layout => undef };
-    }
-    else {
-        header( 'Content-Type' => 'text/comma-separated-values' );
-        template 'ajax/report/phonesdiscovered_csv.tt',
-            { results => \@results },
-            { layout  => undef };
-    }
-};
-
-1;

Netdisco/lib/App/Netdisco/Web/Plugin/Search/Port.pm

@@ -37,6 +37,8 @@ get '/ajax/content/search/port' => require_login sub {
                         ? { "me.mac" => $q }
                         : \[ 'me.mac::text ILIKE ?', $likeval ]
                     ),
+                    { "me.remote_id"   => $likeclause },
+                    { "me.remote_type" => $likeclause },
                 ]
             },
             {   '+columns' => [qw/ device.dns device.name port_vlans.vlan /],

Netdisco/lib/App/Netdisco/Web/Report.pm

@@ -8,7 +8,7 @@ get '/report/*' => require_login sub {
     my ($tag) = splat;
 
     # used in the report search sidebar to populate select inputs
-    my ( $domain_list, $class_list, $ssid_list, $vendor_list );
+    my ( $domain_list, $class_list, $ssid_list, $type_list, $vendor_list );
 
     if ( $tag eq 'netbios' ) {
         $domain_list = [ schema('netdisco')->resultset('NodeNbt')
@@ -31,6 +31,10 @@ get '/report/*' => require_login sub {
         $ssid_list = [ schema('netdisco')->resultset('DevicePortSsid')
                 ->get_distinct_col('ssid') ];
     }
+    elsif ( $tag eq 'nodesdiscovered' ) {
+        $type_list = [ schema('netdisco')->resultset('DevicePort')
+                ->get_distinct_col('remote_type') ];
+    }
     elsif ( $tag eq 'nodevendor' ) {
         $vendor_list = [
             schema('netdisco')->resultset('Node')->search(
@@ -54,6 +58,7 @@ get '/report/*' => require_login sub {
         domain_list => $domain_list,
         class_list  => $class_list,
         ssid_list   => $ssid_list,
+        type_list   => $type_list,
         vendor_list => $vendor_list,
         };
 };