135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into em-device-ports-json

Browse Source 
Conflicts:
	Netdisco/lib/App/Netdisco/Web/Device.pm
This commit is contained in:
Oliver Gorwits
2014-11-16 21:59:35 +00:00
parent dd6904cb18 f6ae7be611
commit 269cbf8708
26 changed files with 179 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
Netdisco/Changes
View File

@@ -1,9 +1,24 @@
2.029013 -
2.029013_002 - 2014-11-14
[ENHANCEMENTS]
* [#161] Updated IOS-XR SSHCollector
* [#165] Mention system clock in docs
* [#164] Workers should restart periodically
* [#168] Jobs requested via web UI are treated as high priority
* [#162] Change from Net::MAC to NetAddr::MAC
* [#159] Macsuck archive behaviour same as ND1 (unseen nodes remain active)
* [#170] Show device SNMP::Info class in web interface
* Add "Run Expire Job" to the Admin Menu
[BUG FIXES]
* Fix for latest DBIx::Class (deploy)
* Fix for latest Dancer (YAML::XS)
* [#160] Job Queue fatal error on num_slots
* [#157] Device Port Log being emptied by device discover
* [#156] Only delete node_ip and node_nbt when no active nodes reference
* [#169] Remove ref to force install of Dancer and DBIC
2.029012 - 2014-10-09

4
Netdisco/META.yml
View File

@@ -59,8 +59,8 @@ requires:
Net::DNS: 0.72
Net::Domain: 1.23
Net::LDAP: 0
Net::MAC: 2.103622
NetAddr::IP: 4.068
NetAddr::MAC: 0.87
Opcode: 1.07
Path::Class: 0.32
Plack: 1.0023
@@ -91,4 +91,4 @@ resources:
homepage: http://netdisco.org/
license: http://opensource.org/licenses/bsd-license.php
repository: git://git.code.sf.net/p/netdisco/netdisco-ng
version: 2.029012
version: 2.029013_002

2
Netdisco/Makefile.PL
View File

@@ -38,7 +38,7 @@ requires 'MCE' => 1.515;
requires 'Net::Domain' => 1.23;
requires 'Net::DNS' => 0.72;
requires 'Net::LDAP' => 0;
requires 'Net::MAC' => 2.103622;
requires 'NetAddr::MAC' => 0.87;
requires 'NetAddr::IP' => 4.068;
requires 'Opcode' => 1.07;
requires 'Path::Class' => 0.32;

9
Netdisco/lib/App/Netdisco.pm
View File

@@ -4,7 +4,7 @@ use strict;
use warnings;
use 5.010_000;
our $VERSION = '2.029012';
our $VERSION = '2.029013_002';
use App::Netdisco::Configuration;
use Module::Find ();
@@ -75,7 +75,8 @@ On Fedora/Red-Hat:
root:~# yum install perl-core perl-DBD-Pg net-snmp-perl make automake gcc
With those installed, we can proceed...
With those installed, next check that your system's clock is correct. Then, we
can proceed...
Create a user on your system called C<netdisco> if one does not already exist.
We'll install Netdisco and its dependencies into this user's home area, which
@@ -123,7 +124,6 @@ install Netdisco and its dependencies into the C<netdisco> user's home area
su - netdisco
curl -L http://cpanmin.us/ | perl - --notest --local-lib ~/perl5 App::Netdisco
~/bin/localenv cpanm --notest --force Dancer@1.3126 DBIx::Class@0.08270
Link some of the newly installed apps into a handy location:
@@ -203,9 +203,6 @@ Notes|App::Netdisco::Manual::ReleaseNotes>. Then, the process is as follows:
# upgrade Netdisco
~/bin/localenv cpanm --notest App::Netdisco
# workaround for current upstream bug
~/bin/localenv cpanm --notest --force Dancer@1.3126 DBIx::Class@0.08270
# apply database schema updates
~/bin/netdisco-deploy

16
Netdisco/lib/App/Netdisco/Core/Discover.pm
View File

@@ -10,7 +10,7 @@ use NetAddr::IP::Lite ':lower';
use List::MoreUtils ();
use Encode;
use Try::Tiny;
use Net::MAC;
use NetAddr::MAC;
use base 'Exporter';
our @EXPORT = ();
@@ -754,9 +754,9 @@ sub store_neighbors {
$device->ip, $remote_ip, $port, $remote_id;
if (!defined $neigh) {
my $mac = Net::MAC->new(mac => $remote_id, 'die' => 0, verbose => 0);
if (not $mac->get_error) {
$neigh = $devices->single({mac => $mac->as_IEEE()});
my $mac = NetAddr::MAC->new(mac => $remote_id);
if ($mac and not $mac->errstr) {
$neigh = $devices->single({mac => $mac->as_microsoft()});
}
}
@@ -765,13 +765,13 @@ sub store_neighbors {
# "myswitchname(012345-012345)"
if (!defined $neigh) {
(my $tmpid = $remote_id) =~ s/.([0-9a-f]{6})-([0-9a-f]{6})./$1$2/;
my $mac = Net::MAC->new(mac => $tmpid, 'die' => 0, verbose => 0);
my $mac = NetAddr::MAC->new(mac => $tmpid);
if (not $mac->get_error) {
if ($mac and not $mac->errstr) {
info sprintf
'[%s] neigh - found neighbor %s by MAC %s',
$device->ip, $remote_id, $mac->as_IEEE();
$neigh = $devices->single({mac => $mac->as_IEEE()});
$device->ip, $remote_id, $mac->as_microsoft();
$neigh = $devices->single({mac => $mac->as_microsoft()});
}
}

9
Netdisco/lib/App/Netdisco/Core/Macsuck.pm
View File

@@ -112,10 +112,15 @@ sub do_macsuck {
$ip, $total_nodes;
# a use for $now ... need to archive dissapeared nodes
my $archived = schema('netdisco')->resultset('Node')->search({
my $archived = 0;
if (setting('node_freshness')) {
$archived = schema('netdisco')->resultset('Node')->search({
switch => $ip,
time_last => { '<' => \$now },
time_last => \[ "< ($now - ?::interval)",
setting('node_freshness') .' minutes' ],
})->update({ active => \'false' });
}
debug sprintf ' [%s] macsuck - removed %d fwd table entries to archive',
$ip, $archived;

6
Netdisco/lib/App/Netdisco/DB/Result/DevicePort.pm
View File

@@ -7,7 +7,7 @@ package App::Netdisco::DB::Result::DevicePort;
use strict;
use warnings;
use Net::MAC;
use NetAddr::MAC;
use MIME::Base64 'encode_base64url';
@@ -339,10 +339,10 @@ sub base64url_port { return encode_base64url((shift)->port) }
=head2 net_mac
Returns the C<mac> column instantiated into a L<Net::MAC> object.
Returns the C<mac> column instantiated into a L<NetAddr::MAC> object.
=cut
sub net_mac { return Net::MAC->new(mac => (shift)->mac) }
sub net_mac { return NetAddr::MAC->new(mac => (shift)->mac) }
1;

6
Netdisco/lib/App/Netdisco/DB/Result/Node.pm
View File

@@ -7,7 +7,7 @@ package App::Netdisco::DB::Result::Node;
use strict;
use warnings;
use Net::MAC;
use NetAddr::MAC;
use base 'DBIx::Class::Core';
__PACKAGE__->table("node");
@@ -175,10 +175,10 @@ sub time_last_stamp { return (shift)->get_column('time_last_stamp') }
=head2 net_mac
Returns the C<mac> column instantiated into a L<Net::MAC> object.
Returns the C<mac> column instantiated into a L<NetAddr::MAC> object.
=cut
sub net_mac { return Net::MAC->new(mac => (shift)->mac) }
sub net_mac { return NetAddr::MAC->new(mac => (shift)->mac) }
1;

6
Netdisco/lib/App/Netdisco/DB/Result/NodeIp.pm
View File

@@ -7,7 +7,7 @@ package App::Netdisco::DB::Result::NodeIp;
use strict;
use warnings;
use Net::MAC;
use NetAddr::MAC;
use base 'DBIx::Class::Core';
__PACKAGE__->table("node_ip");
@@ -221,10 +221,10 @@ sub time_last_stamp { return (shift)->get_column('time_last_stamp') }
=head2 net_mac
Returns the C<mac> column instantiated into a L<Net::MAC> object.
Returns the C<mac> column instantiated into a L<NetAddr::MAC> object.
=cut
sub net_mac { return Net::MAC->new(mac => (shift)->mac) }
sub net_mac { return NetAddr::MAC->new(mac => (shift)->mac) }
1;

6
Netdisco/lib/App/Netdisco/DB/Result/NodeNbt.pm
View File

@@ -7,7 +7,7 @@ package App::Netdisco::DB::Result::NodeNbt;
use strict;
use warnings;
use Net::MAC;
use NetAddr::MAC;
use base 'DBIx::Class::Core';
__PACKAGE__->table("node_nbt");
@@ -178,10 +178,10 @@ sub time_last_stamp { return (shift)->get_column('time_last_stamp') }
=head2 net_mac
Returns the C<mac> column instantiated into a L<Net::MAC> object.
Returns the C<mac> column instantiated into a L<NetAddr::MAC> object.
=cut
sub net_mac { return Net::MAC->new(mac => (shift)->mac) }
sub net_mac { return NetAddr::MAC->new(mac => (shift)->mac) }
1;

6
Netdisco/lib/App/Netdisco/DB/Result/NodeWireless.pm
View File

@@ -7,7 +7,7 @@ package App::Netdisco::DB::Result::NodeWireless;
use strict;
use warnings;
use Net::MAC;
use NetAddr::MAC;
use base 'DBIx::Class::Core';
__PACKAGE__->table("node_wireless");
@@ -87,10 +87,10 @@ __PACKAGE__->belongs_to( node => 'App::Netdisco::DB::Result::Node',
=head2 net_mac
Returns the C<mac> column instantiated into a L<Net::MAC> object.
Returns the C<mac> column instantiated into a L<NetAddr::MAC> object.
=cut
sub net_mac { return Net::MAC->new(mac => (shift)->mac) }
sub net_mac { return NetAddr::MAC->new(mac => (shift)->mac) }
1;

1
Netdisco/lib/App/Netdisco/DB/ResultSet/DevicePort.pm
View File

@@ -157,7 +157,6 @@ sub delete {
DevicePortVlan
DevicePortWireless
DevicePortSsid
DevicePortLog
/) {
$schema->resultset($set)->search(
{ ip => { '-in' => $ports->as_query }},

21
Netdisco/lib/App/Netdisco/DB/ResultSet/Node.pm
View File

@@ -109,9 +109,30 @@ sub delete {
return 0E0;
}
else {
# for node_ip and node_nbt *only* delete if there are no longer
# any active nodes referencing the IP or NBT (hence 2nd IN clause).
foreach my $set (qw/
NodeIp
NodeNbt
/) {
$schema->resultset($set)->search({
'-and' => [
'me.mac' => { '-in' => $nodes->as_query },
'me.mac' => { '-in' => $schema->resultset($set)->search({
-bool => 'nodes.active',
},
{
columns => 'mac',
join => 'nodes',
group_by => 'me.mac',
having => \[ 'count(nodes.mac) = 0' ],
})->as_query,
},
],
})->delete;
}
foreach my $set (qw/
NodeMonitor
NodeWireless
/) {

10
Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm
View File

@@ -10,6 +10,8 @@ use namespace::clean;
use App::Netdisco::JobQueue qw/jq_defer jq_complete/;
sub worker_begin { (shift)->{started} = time }
sub worker_body {
my $self = shift;
my $wid = $self->wid;
@@ -38,6 +40,14 @@ sub worker_body {
};
$self->close_job($job);
# restart worker once a day.
# relies on the worker seeing a job at least every hour.
my $hour = [localtime()]->[2];
if ($wid and (time >= ($self->{started} + 86400))
and ($hour == ($wid % 24))) {
$self->exit(0, "recycling worker $wid");
}
}
}

32
Netdisco/lib/App/Netdisco/JobQueue/PostgreSQL.pm
View File

@@ -25,26 +25,40 @@ our @EXPORT_OK = qw/
/;
our %EXPORT_TAGS = ( all => \@EXPORT_OK );
sub jq_getsome {
my ($num_slots, $prio) = @_;
return () if defined $num_slots and $num_slots eq '0';
$num_slots ||= 1;
$prio ||= 'normal';
my @returned = ();
sub _getsome {
my ($num_slots, $where) = @_;
return () if ((!defined $num_slots) or ($num_slots < 1));
return () if ((!defined $where) or (ref {} ne ref $where));
my $rs = schema('netdisco')->resultset('Admin')
->search(
{status => 'queued', action => { -in => setting('job_prio')->{$prio} } },
{order_by => 'random()', rows => ($num_slots || 1)},
{ status => 'queued', %$where },
{ order_by => 'random()', rows => $num_slots },
);
my @returned = ();
while (my $job = $rs->next) {
push @returned, App::Netdisco::Daemon::Job->new({ $job->get_columns });
}
return @returned;
}
sub jq_getsomep { return jq_getsome(shift, 'high') }
sub jq_getsome {
return _getsome(shift,
{ action => { -in => setting('job_prio')->{'normal'} } }
);
}
sub jq_getsomep {
return _getsome(shift, {
-or => [{
username => { '!=' => undef },
action => { -in => setting('job_prio')->{'normal'} },
},{
action => { -in => setting('job_prio')->{'high'} },
}],
});
}
sub jq_locked {
my $fqdn = hostfqdn || 'localhost';

15
Netdisco/lib/App/Netdisco/Manual/Configuration.pod
View File

@@ -741,6 +741,21 @@ Value: Number. Default: 1.
Seconds nbtstat will wait for a response before time out. Accepts fractional
seconds as well as integers.
=head3 C<node_freshness>
Value: Number of Minutes. Default: 0
Controls the behaviour of Netdisco when a node (workstation, printer, etc) has
disappeared from the network (device MAC address tables).
If set to 0, the default, nodes will remain on the last-seen switch port until
"C<expire_nodes>" days have passed (when they'll be deleted if you run the
Expire job). This is the same behaviour as Netdisco 1.
Set to a number of minutes to enforce some kind of ageing on this data. For
example you could set to 60 to match the default macsuck schedule, meaning
nodes are archived if they're not in the device tables at the time of polling.
=head3 C<expire_devices>
Value: Number of Days. Default: 60

2
Netdisco/lib/App/Netdisco/Manual/Developing.pod
View File

@@ -259,7 +259,7 @@ Compared to the current Netdisco, the handler routines are very small. This is
because (a) they don't include any HTML - this is delegated to a template, and
(b) they don't include an SQL - this is delegated to DBIx::Class. Small
routines are more manageable, and easier to maintain. You'll also notice use
of modules such as L<Net::MAC> and L<NetAddr::IP::Lite> to simplify and make
of modules such as L<NetAddr::MAC> and L<NetAddr::IP::Lite> to simplify and make
more robust the handling of data.
In fact, many sections of the web application have been factored out into

9
Netdisco/lib/App/Netdisco/Manual/ReleaseNotes.pod
View File

@@ -36,6 +36,15 @@ but they are backwards compatible.
=back
=head1 2.029013_002
=head2 General Notices
The node archiving behaviour of Netdisco 2 has until now been accidentally
different to that in Netdisco 1. This has now been fixed. See the new
"C<node_freshness>" configuration setting if you wish to revert or tune this
behaviour.
=head1 2.029010
=head2 General Notices

24
Netdisco/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm
View File

@@ -16,7 +16,6 @@ use strict;
use warnings;
use Dancer ':script';
use Expect;
use Moo;
=head1 PUBLIC METHODS
@@ -36,27 +35,18 @@ sub arpnip {
my ($self, $hostlabel, $ssh, @args) = @_;
debug "$hostlabel $$ arpnip()";
my @data = $ssh->capture("show arp vrf all");
my ($pty, $pid) = $ssh->open2pty or die "unable to run remote command";
my $expect = Expect->init($pty);
my ($pos, $error, $match, $before, $after);
my $prompt = qr/#/;
($pos, $error, $match, $before, $after) = $expect->expect(10, -re, $prompt);
$expect->send("terminal length 0\n");
($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
chomp @data;
my @arpentries;
$expect->send("show arp vrf all\n");
($pos, $error, $match, $before, $after) = $expect->expect(5, -re, $prompt);
# 0.0.0.0 00:00:00 0000.0000.0000 Dynamic ARPA GigabitEthernet0/0/0/0
for (split(/\n/, $before)){
foreach (@data) {
my ($ip, $age, $mac, $state, $t, $iface) = split(/\s+/);
if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/ && $mac =~ m/[0-9a-f.]+/i) {
if ($ip =~ m/(\d{1,3}\.){3}\d{1,3}/
&& $mac =~ m/([0-9a-f]{4}\.){2}[0-9a-f]{4}/i) {
push(@arpentries, { ip => $ip, mac => $mac });
}
}

16
Netdisco/lib/App/Netdisco/Util/Node.pm
View File

@@ -3,7 +3,7 @@ package App::Netdisco::Util::Node;
use Dancer qw/:syntax :script/;
use Dancer::Plugin::DBIC 'schema';
use Net::MAC;
use NetAddr::MAC;
use App::Netdisco::Util::Permission 'check_acl';
use base 'Exporter';
@@ -66,23 +66,23 @@ MAC address does not belong to an interface on any known Device
sub check_mac {
my ($device, $node, $port_macs) = @_;
my $mac = Net::MAC->new(mac => $node, 'die' => 0, verbose => 0);
my $mac = NetAddr::MAC->new(mac => $node);
my $devip = (ref $device ? $device->ip : '');
$port_macs ||= {};
# incomplete MAC addresses (BayRS frame relay DLCI, etc)
if ($mac->get_error) {
if (!defined $mac or $mac->errstr) {
debug sprintf ' [%s] check_mac - mac [%s] malformed - skipping',
$devip, $node;
return 0;
}
else {
# lower case, hex, colon delimited, 8-bit groups
$node = lc $mac->as_IEEE;
$node = lc $mac->as_microsoft;
}
# broadcast MAC addresses
return 0 if $node eq 'ff:ff:ff:ff:ff:ff';
return 0 if $mac->is_broadcast;
# all-zero MAC addresses
return 0 if $node eq '00:00:00:00:00:00';
@@ -91,21 +91,21 @@ sub check_mac {
return 0 if $node eq '00:00:00:00:00:01';
# multicast
if ($node =~ m/^[0-9a-f](?:1|3|5|7|9|b|d|f):/) {
if ($mac->is_multicast and not $mac->is_msnlb) {
debug sprintf ' [%s] check_mac - multicast mac [%s] - skipping',
$devip, $node;
return 0;
}
# VRRP
if (index($node, '00:00:5e:00:01:') == 0) {
if ($mac->is_vrrp) {
debug sprintf ' [%s] check_mac - VRRP mac [%s] - skipping',
$devip, $node;
return 0;
}
# HSRP
if (index($node, '00:00:0c:07:ac:') == 0) {
if ($mac->is_hsrp or $mac->is_hsrp2) {
debug sprintf ' [%s] check_mac - HSRP mac [%s] - skipping',
$devip, $node;
return 0;

8
Netdisco/lib/App/Netdisco/Web/Plugin/Search/Node.pm
View File

@@ -6,7 +6,7 @@ use Dancer::Plugin::DBIC;
use Dancer::Plugin::Auth::Extensible;
use NetAddr::IP::Lite ':lower';
use Net::MAC ();
use NetAddr::MAC ();
use App::Netdisco::Web::Plugin;
use App::Netdisco::Util::Web 'sql_match';
@@ -20,9 +20,9 @@ ajax '/ajax/content/search/node' => require_login sub {
content_type('text/html');
my $agenot = param('age_invert') || '0';
my ( $start, $end ) = param('daterange') =~ /(\d+-\d+-\d+)/gmx;
my ( $start, $end ) = param('daterange') =~ m/(\d+-\d+-\d+)/gmx;
my $mac = Net::MAC->new(mac => $node, 'die' => 0, verbose => 0);
my $mac = NetAddr::MAC->new(mac => $node);
my @active = (param('archived') ? () : (-bool => 'active'));
my @times = ();
@@ -48,7 +48,7 @@ ajax '/ajax/content/search/node' => require_login sub {
my @where_mac =
($using_wildcards ? \['me.mac::text ILIKE ?', $likeval]
: ($mac->get_error ? \'0=1' : ('me.mac' => $mac->as_IEEE)) );
: ((!defined $mac or $mac->errstr) ? \'0=1' : ('me.mac' => $mac->as_microsoft)) );
my $sightings = schema('netdisco')->resultset('Node')
->search({-and => [@where_mac, @active, @times]}, {

1
Netdisco/share/config.yml
View File

@@ -198,6 +198,7 @@ nbtstat_only: []
nbtstat_max_age: 7
nbtstat_interval: 0.02
nbtstat_timeout: 1
node_freshness: 0
expire_devices: 60
expire_nodes: 90
expire_nodes_archive: 60

10
Netdisco/share/views/ajax/device/details.tt
View File

@@ -83,14 +83,18 @@
<tr>
<td>Administration</td>
<td>
<a href="ssh://[% d.ip | html_entity %]" target="_blank">
<a href="ssh://[% d.ip | uri %]" target="_blank">
<span class="label label-info"><i class="icon-keyboard"></i> SSH</span></a>
<a href="telnet://[% d.ip | html_entity %]" target="_blank">
<a href="telnet://[% d.ip | uri %]" target="_blank">
<span class="label label-info"><i class="icon-keyboard"></i> Telnet</span></a>
<a href="https://[% d.ip | html_entity %]/" target="_blank">
<a href="https://[% d.ip | uri %]/" target="_blank">
<span class="label label-info"><i class="icon-external-link"></i> Web</span></a>
</td>
</tr>
<tr>
<td>SNMP Class</td>
<td><a target="_blank" href="https://metacpan.org/pod/[% d.snmp_class | uri %]">[% d.snmp_class | html_entity %]</td>
</tr>
<tr>
<td>Uptime</td>
<td>[% d.uptime_age | html_entity %]</td>

5
Netdisco/share/views/layouts/main.tt
View File

@@ -122,6 +122,11 @@
<button type="submit" class="btn btn-link nd_btn-link">NBTstat All</button>
</form>
</li>
<li>
<form method="post" class="nd_inline-form" action="[% uri_for('/admin/expire') %]">
<button type="submit" class="btn btn-link nd_btn-link">Run Expire Job</button>
</form>
</li>
[% IF settings._admin_tasks.size %]
<li class="divider"></li>
[% FOREACH ai IN settings._admin_order %]

7
Netdisco/share/views/sidebar/device/ports.tt
View File

@@ -128,9 +128,10 @@
<li>
<em class="muted">MAC address format:</em><br/>
<select id="nd_mac-format" name="mac_format">
[% FOREACH format IN [ 'IEEE', 'Cisco', 'Microsoft', 'Sun' ] %]
<option[% ' selected="selected"' IF params.mac_format == format %]>[% format %]</option>
[% END %]
<option[% ' selected="selected"' IF params.mac_format == 'Cisco' %]>Cisco</option>
<option[% ' selected="selected"' IF params.mac_format == 'Microsoft' %] value="Microsoft">IEEE</option>
<option[% ' selected="selected"' IF params.mac_format == 'IEEE' %] value="IEEE">Microsoft</option>
<option[% ' selected="selected"' IF params.mac_format == 'Sun' %]>Sun</option>
</select>
</li>
<li>

7
Netdisco/share/views/sidebar/search/node.tt
View File

@@ -55,9 +55,10 @@
<div class="clearfix">
<em class="muted">MAC address format:</em><br/>
<select id="nd_node-mac-format" name="mac_format">
[% FOREACH format IN [ 'IEEE', 'Cisco', 'Microsoft', 'Sun' ] %]
<option[% ' selected="selected"' IF params.mac_format == format %]>[% format %]</option>
[% END %]
<option[% ' selected="selected"' IF params.mac_format == 'Cisco' %]>Cisco</option>
<option[% ' selected="selected"' IF params.mac_format == 'Microsoft' %] value="Microsoft">IEEE</option>
<option[% ' selected="selected"' IF params.mac_format == 'IEEE' %] value="IEEE">Microsoft</option>
<option[% ' selected="selected"' IF params.mac_format == 'Sun' %]>Sun</option>
</select>
</div>
<button id="[% tab.tag %]_submit" type="submit" class="btn btn-info">