135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#910 implement import of snmpwalk and more robust snapshot handling (#1086)

Browse Source 
* initial work

* initial work

* initial work

* some fixes for time and Layer3 weird spec

* store again the snapshot after update for specific

* resolve the enums

* monkeypatch SNMP::translateObj to avoid hardware exception on macOS

* only save cache to db in the late phase worker

* no need to check for cache on transport, can just go ahead and try

* use database only for oidmap instead of netdisco-mibs

* rewrite device snapshot to gather loaded mib leafs only

* remove old walker code from snapshot worker

* allow snmp browser to work without snapshot

* only store snapshot leafs which the device responded on

* refactor to separate snapshot work from snmp transport work

* refactor to separate snapshot work from snmp transport work

* allow typeahead on MIB qualified leafs

* fixes for snmpwalk input after previous refactor

* add the extra stuff SNMP::Info device class uses into snapshot

* better width for snmp search box

* fix css for snmp options

* add spinner while snmp loading

* add spinner while snmp loading

* add spinner while snmp loading

* support SNMP::Info device class or named MIBs as extra on snapshot

* add final tidy and bug fix
This commit is contained in:
Oliver Gorwits
2023-08-10 22:27:02 +01:00
committed by GitHub
parent 7afae0b9b2
commit 9eb537a4c1
12 changed files with 803 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
lib/App/Netdisco/Configuration.pm
View File

@@ -19,6 +19,21 @@ BEGIN {
}
}
BEGIN {
no warnings 'redefine';
use SNMP;
# hardware exception on macOS at least when translateObj
# gets something like '.0.0' passed as arg
my $orig_translate = *SNMP::translateObj{'CODE'};
*SNMP::translateObj = sub {
my $arg = $_[0];
return undef unless defined $arg and $arg !~ m/^[.0]+$/;
return $orig_translate->(@_);
};
}
# set up database schema config from simple config vars
if (ref {} eq ref setting('database')) {
# override from env for docker

2
lib/App/Netdisco/DB/Result/DeviceBrowser.pm
View File

@@ -47,4 +47,6 @@ __PACKAGE__->belongs_to(
{ join_type => 'RIGHT' }
);
__PACKAGE__->belongs_to( oid_fields => 'App::Netdisco::DB::Result::SNMPObject', 'oid' );
1;

30
lib/App/Netdisco/Transport/SNMP.pm
View File

@@ -6,16 +6,12 @@ use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Util::SNMP 'get_communities';
use App::Netdisco::Util::Device 'get_device';
use App::Netdisco::Util::Permission 'acl_matches';
use App::Netdisco::Util::Snapshot qw/load_cache_for_device add_snmpinfo_aliases/;
use SNMP::Info;
use Try::Tiny;
use Module::Load ();
use Storable 'thaw';
use File::Slurper 'read_text';
use MIME::Base64 'decode_base64';
use Path::Class 'dir';
use File::Path 'make_path';
use File::Spec::Functions qw(catdir catfile);
use NetAddr::IP::Lite ':lower';
use List::Util qw/pairkeys pairfirst/;
@@ -64,12 +60,6 @@ sub reader_for {
my ($class, $ip, $useclass) = @_;
my $device = get_device($ip) or return undef;
my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip );
if ($device->in_storage and $device->is_pseudo and ! -f $pseudo_cache) {
error sprintf 'transport error - cannot act on pseudo-device [%s] without offline cache', $device->ip;
return undef;
}
my $readers = $class->instance->readers or return undef;
return $readers->{$device->ip} if exists $readers->{$device->ip};
@@ -171,13 +161,13 @@ sub _snmp_connect_generic {
}
# support for offline cache
my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip );
if (-f $pseudo_cache and ($device->is_pseudo or ! $device->in_storage)) {
$snmp_args{Cache} = thaw( decode_base64( read_text($pseudo_cache) ) );
$snmp_args{Offline} = 1;
# support pseudo/offline device renumber and also pseudo device autovivification
$device->set_column(is_pseudo => \'true') if ! $device->is_pseudo;
debug sprintf 'snmp transport running in offline mode for: [%s]', $device->ip;
my $cache = load_cache_for_device($device);
if (scalar keys %$cache) {
$snmp_args{Cache} = $cache;
$snmp_args{Offline} = 1;
# support pseudo/offline device renumber and also pseudo device autovivification
$device->set_column(is_pseudo => \'true') if not $device->is_pseudo;
debug sprintf 'snmp transport running in offline mode for: [%s]', $device->ip;
}
# any net-snmp options to add or override
@@ -229,6 +219,7 @@ sub _snmp_connect_generic {
my $class = $info->device_type;
return $class->new(
%snmp_args, Version => $ver,
($info->offline ? (Cache => $info->cache) : ()),
_mk_info_commargs($comm),
);
}
@@ -298,6 +289,7 @@ sub _try_connect {
Module::Load::load $class;
$info = $class->new(%$snmp_args, %comm_args);
add_snmpinfo_aliases($info) if $info->offline;
}
}
catch {
@@ -388,7 +380,7 @@ sub _build_mibdirs {
sub _get_mibdirs_content {
my $home = shift;
my @list = map {s|$home/||; $_} grep {m/[a-z0-9]/} grep {-d} glob("$home/*");
my @list = map {s|$home/||; $_} grep { m|/[a-z0-9-]+$| } grep {-d} glob("$home/*");
return \@list;
}

3
lib/App/Netdisco/Util/Device.pm
View File

@@ -166,9 +166,8 @@ sub is_discoverable {
$remote_type ||= '';
$remote_cap ||= [];
my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip );
return _bail_msg("is_discoverable: $device is pseudo-device without offline cache")
if $device->is_pseudo and ! -f $pseudo_cache;
if $device->is_pseudo and not $device->oids->count;
return _bail_msg("is_discoverable: $device matches wap_platforms but discover_waps is not enabled")
if ((not setting('discover_waps')) and

46
lib/App/Netdisco/Util/SNMP.pm
View File

@@ -3,8 +3,10 @@ package App::Netdisco::Util::SNMP;
use Dancer qw/:syntax :script !to_json !from_json/;
use App::Netdisco::Util::DeviceAuth 'get_external_credentials';
use MIME::Base64 'decode_base64';
use Storable 'thaw';
use File::Spec::Functions qw/splitdir catdir catfile/;
use MIME::Base64 qw/decode_base64/;
use Storable qw/thaw/;
use SNMP::Info;
use JSON::PP;
use base 'Exporter';
@@ -12,9 +14,9 @@ our @EXPORT = ();
our @EXPORT_OK = qw/
get_communities
snmp_comm_reindex
sortable_oid
decode_and_munge
%ALL_MUNGERS
decode_and_munge
sortable_oid
/;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
@@ -31,24 +33,6 @@ subroutines.
=head1 EXPORT_OK
=head2 sortable_oid( $oid, $seglen? )
Take an OID and return a version of it which is sortable using C<cmp>
operator. Works by zero-padding the numeric parts all to be length
C<< $seglen >>, which defaults to 6.
=cut
# take oid and make comparable
sub sortable_oid {
my ($oid, $seglen) = @_;
$seglen ||= 6;
return $oid if $oid !~ m/^[0-9.]+$/;
$oid =~ s/^(\.)//; my $leading = $1;
$oid = join '.', map { sprintf("\%0${seglen}d", $_) } (split m/\./, $oid);
return (($leading || '') . $oid);
}
=head2 get_communities( $device, $mode )
Takes the current C<device_auth> setting and pushes onto the front of the list
@@ -247,4 +231,22 @@ sub decode_and_munge {
}
=head2 sortable_oid( $oid, $seglen? )
Take an OID and return a version of it which is sortable using C<cmp>
operator. Works by zero-padding the numeric parts all to be length
C<< $seglen >>, which defaults to 6.
=cut
# take oid and make comparable
sub sortable_oid {
my ($oid, $seglen) = @_;
$seglen ||= 6;
return $oid if $oid !~ m/^[0-9.]+$/;
$oid =~ s/^(\.)//; my $leading = $1;
$oid = join '.', map { sprintf("\%0${seglen}d", $_) } (split m/\./, $oid);
return (($leading || '') . $oid);
}
true;

622
lib/App/Netdisco/Util/Snapshot.pm Normal file
View File

@@ -0,0 +1,622 @@
package App::Netdisco::Util::Snapshot;
use Dancer qw/:syntax :script !to_json !from_json/;
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Util::SNMP 'sortable_oid';
use File::Spec::Functions qw/splitdir catdir catfile/;
use MIME::Base64 qw/encode_base64 decode_base64/;
use File::Slurper qw/read_lines read_text/;
use Sub::Util 'subname';
use Storable qw/dclone nfreeze thaw/;
use Scalar::Util 'blessed';
use Module::Load ();
use SNMP::Info;
use base 'Exporter';
our @EXPORT = ();
our @EXPORT_OK = qw/
load_cache_for_device
snmpwalk_to_cache
gather_every_mib_object
dump_cache_to_browserdata
add_snmpinfo_aliases
get_oidmap_from_database
get_oidmap_from_mibs_files
get_mibs_for
get_munges
/;
our %EXPORT_TAGS = (all => \@EXPORT_OK);
=head1 NAME
App::Netdisco::Util::Snapshot
=head1 DESCRIPTION
Helper functions for L<SNMP::Info> instances.
There are no default exports, however the C<:all> tag will export all
subroutines.
=head1 EXPORT_OK
=head2 load_cache_for_device( $device )
Tries to find a device cache in database or on disk, or build one from
a net-snmp snmpwalk on disk. Returns a cache.
=cut
sub load_cache_for_device {
my $device = shift;
return {} unless ($device->is_pseudo or not $device->in_storage);
# ideally we have a cache in the db
if ($device->is_pseudo and my $snapshot = $device->snapshot) {
return thaw( decode_base64( $snapshot->cache ) );
}
# or we have a file on disk - could be cache or walk
my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip );
if (-f $pseudo_cache and not $device->in_storage) {
my $content = read_text($pseudo_cache);
if ($content =~ m/^\.1/) {
my %oids = ();
# parse the snmpwalk output which looks like
# .1.0.8802.1.1.2.1.1.1.0 = INTEGER: 30
my @lines = split /\n/, $content;
foreach my $line (@lines) {
my ($oid, $val) = $line =~ m/^(\S+) = (?:[^:]+: )?(.+)$/;
next unless $oid and $val;
# empty string makes the capture go wonky
$val = '' if $val =~ m/^[^:]+: ?$/;
# remove quotes from strings
$val =~ s/^"//;
$val =~ s/"$//;
$oids{$oid} = $val;
}
return snmpwalk_to_cache(%oids);
}
else {
my $cache = thaw( decode_base64( $content ) );
return add_snmpinfo_aliases( $cache );
}
# there is a late phase discover worker to generate the oids
# and also to save the cache into the database, because we want
# to wait for device-specific SNMP::Info class and all its methods.
}
return {};
}
=head2 snmpwalk_to_cache ( %oids )
Take the snmpwalk of the device which is numeric (no MIB translateObj),
resolve to MIB identifiers using netdisco-mibs data, then return as an
SNMP::Info instance cache.
=cut
sub snmpwalk_to_cache {
my %oids = @_;
return () unless scalar keys %oids;
my %oidmap = reverse get_oidmap_from_database();
my %leaves = ();
OID: foreach my $orig_oid (keys %oids) {
my $oid = $orig_oid;
my $idx = '';
while (length($oid) and !exists $oidmap{$oid}) {
$oid =~ s/\.(\d+)$//;
$idx = ((defined $idx and length $idx) ? "${1}.${idx}" : $1);
}
if (exists $oidmap{$oid}) {
$idx =~ s/^\.//;
my $qleaf = $oidmap{$oid};
my $key = $oid .'~~'. $qleaf;
if ($idx eq 0) {
$leaves{$key} = $oids{$orig_oid};
}
else {
# on rare occasions a vendor returns .0 and .something
delete $leaves{$key}
if defined $leaves{$key} and ref q{} eq $leaves{$key};
$leaves{$key}->{$idx} = $oids{$orig_oid};
}
# debug "snapshot $device - cached $oidmap{$oid}($idx) from $orig_oid";
next OID;
}
# this is not too surprising
# debug sprintf "cache builder - error: missing OID %s in netdisco-mibs", $orig_oid;
}
my $info = SNMP::Info->new({
Offline => 1,
Cache => {},
AutoSpecify => 0,
Session => {},
});
foreach my $attr (keys %leaves) {
my ($oid, $qleaf) = split m/~~/, $attr;
my $val = $leaves{$attr};
# resolve the enums if needed
my $row = schema('netdisco')->resultset('SNMPObject')->find($oid);
if ($row and $row->enum) {
my %emap = map { reverse split m/\(/ }
map { s/\)//; $_ }
@{ $row->enum };
if (ref q{} eq ref $val) {
$val = $emap{$val} if exists $emap{$val};
}
elsif (ref {} eq ref $val) {
foreach my $k (keys %$val) {
$val->{$k} = $emap{ $val->{$k} }
if exists $emap{ $val->{$k} };
}
}
}
my $leaf = $qleaf;
$leaf =~ s/.+:://;
my $snmpqleaf = $qleaf;
$snmpqleaf =~ s/[-:]/_/g;
# do we need this ?? $info->_cache($oid, $leaves{$attr});
$info->_cache($leaf, $leaves{$attr});
$info->_cache($snmpqleaf, $leaves{$attr});
}
debug sprintf "snmpwalk_to_cache: cache size: %d", scalar keys %{ $info->cache };
# inject a basic set of SNMP::Info globals and funcs aliases
# which are needed for initial device discovery
add_snmpinfo_aliases($info);
return $info->cache;
}
=head2 gather_every_mib_object( $device, $snmp, @extramibs? )
Gathers evey MIB Object in the MIBs loaded for the device and store
to the database for SNMP browser.
Optionally add a list of vendors, MIBs, or SNMP:Info class for extra MIB
Objects from the netdisco-mibs bundle.
The passed SNMP::Info instance has its cache update with the data.
=cut
sub gather_every_mib_object {
my ($device, $snmp, @extra) = @_;
# get MIBs loaded for device
my @mibs = keys %{ $snmp->mibs() };
my @extra_mibs = get_mibs_for(@extra);
debug sprintf "-> covering %d MIBs", (scalar @mibs + scalar @extra_mibs);
SNMP::loadModules($_) for @extra_mibs;
# get qualified leafs for those MIBs from snmp_object
my %oidmap = get_oidmap_from_database(@mibs, @extra_mibs);
debug sprintf "-> gathering %d MIB Objects", scalar keys %oidmap;
foreach my $qleaf (sort {sortable_oid($oidmap{$a}) cmp sortable_oid($oidmap{$b})} keys %oidmap) {
my $leaf = $qleaf;
$leaf =~ s/.+:://;
my $snmpqleaf = $qleaf;
$snmpqleaf =~ s/[-:]/_/g;
# gather the leaf
$snmp->$snmpqleaf;
# skip any leaf which did not return data from device
# this works for both funcs and globals as funcs create stub global
next unless exists $snmp->cache->{'_'. $snmpqleaf};
# store a short name alias which is needed for netdisco actions
$snmp->_cache($leaf, $snmp->$snmpqleaf);
}
}
=head2 dump_cache_to_browserdata( $device, $snmp )
Dumps any valid MIB leaf from the passed SNMP::Info instance's cache into
the Netdisco database SNMP Browser table.
Ideally the leafs are fully qualified, but if not then a best effort will
be made to find their correct MIB.
=cut
sub dump_cache_to_browserdata {
my ($device, $snmp) = @_;
my %qoidmap = get_oidmap_from_database();
my %oidmap = get_leaf_to_qleaf_map();
my %munges = get_munges($snmp);
my $cache = $snmp->cache;
my %oids = ();
foreach my $key (keys %$cache) {
next unless $key and $key =~ m/^_/;
my $snmpqleaf = $key;
$snmpqleaf =~ s/^_//;
my $qleaf = $snmpqleaf;
$qleaf =~ s/__/::/;
$qleaf =~ s/_/-/g;
my $leaf = $qleaf;
$leaf =~ s/.+:://;
next unless exists $qoidmap{$qleaf}
or (exists $oidmap{$leaf} and exists $qoidmap{ $oidmap{$leaf} });
my $oid = exists $qoidmap{$qleaf} ? $qoidmap{$qleaf} : $qoidmap{ $oidmap{$leaf} };
my $data = exists $cache->{'store'}{$snmpqleaf} ? $cache->{'store'}{$snmpqleaf}
: $cache->{$key};
next unless defined $data;
push @{ $oids{$oid} }, {
oid => $oid,
oid_parts => [ grep {length} (split m/\./, $oid) ],
leaf => $leaf,
qleaf => $qleaf,
munge => ($munges{$snmpqleaf} || $munges{$leaf}),
value => encode_base64( nfreeze( [$data] ) ),
};
}
%oids = map { ($_ => [sort {length($b->{qleaf}) <=> length($a->{qleaf})} @{ $oids{$_} }]) }
keys %oids;
schema('netdisco')->txn_do(sub {
my $gone = $device->oids->delete;
debug sprintf '-> removed %d oids from db', $gone;
$device->oids->populate([ sort {sortable_oid($a->{oid}) cmp sortable_oid($b->{oid})}
map { delete $_->{qleaf}; $_ }
map { $oids{$_}->[0] } keys %oids ]);
debug sprintf '-> added %d new oids to db', scalar keys %oids;
});
}
=head2 add_snmpinfo_aliases( $snmp_info_instance | $snmp_info_cache )
Add in any GLOBALS and FUNCS aliases from the SNMP::Info device class
or else a set of defaults that allow device discovery. Returns the cache.
=cut
sub add_snmpinfo_aliases {
my $info = shift or return {};
if (not blessed $info) {
$info = SNMP::Info->new({
Offline => 1,
Cache => $info,
AutoSpecify => 0,
Session => {},
});
}
my %globals = %{ $info->globals };
my %funcs = %{ $info->funcs };
while (my ($alias, $leaf) = each %globals) {
next if $leaf =~ m/\.\d+$/;
$info->_cache($alias, $info->$leaf) if $info->$leaf;
}
while (my ($alias, $leaf) = each %funcs) {
$info->_cache($alias, dclone $info->$leaf) if ref q{} ne ref $info->$leaf;
}
# SNMP::Info::Layer3 has some weird aliases we can fix here
$info->_cache('serial1', $info->chassisId->{''}) if ref {} eq ref $info->chassisId;
$info->_cache('router_ip', $info->ospfRouterId->{''}) if ref {} eq ref $info->ospfRouterId;
$info->_cache('bgp_id', $info->bgpIdentifier->{''}) if ref {} eq ref $info->bgpIdentifier;
$info->_cache('bgp_local_as', $info->bgpLocalAs->{''}) if ref {} eq ref $info->bgpLocalAs;
$info->_cache('sysUpTime', $info->sysUpTimeInstance->{''}) if ref {} eq ref $info->sysUpTimeInstance
and not $info->sysUpTime;
$info->_cache('mac', $info->ifPhysAddress->{1}) if ref {} eq ref $info->ifPhysAddress;
# now for any other SNMP::Info method in GLOBALS or FUNCS which Netdisco
# might call, but will not have data, we fake a cache entry to avoid
# throwing errors
while (my $method = <DATA>) {
$method =~ s/\s//g;
next unless length $method and not $info->$method;
$info->_cache($method, '') if exists $globals{$method};
$info->_cache($method, {}) if exists $funcs{$method};
}
debug sprintf "add_snmpinfo_aliases: cache size: %d", scalar keys %{ $info->cache };
return $info->cache;
}
=head2 get_leaf_to_qleaf_map( )
=cut
sub get_leaf_to_qleaf_map {
debug "-> loading database leaf to qleaf map";
my %oidmap = map { ( $_->{leaf} => (join '::', $_->{mib}, $_->{leaf}) ) }
schema('netdisco')->resultset('SNMPObject')
->search({
num_children => 0,
leaf => { '!~' => 'anonymous#\d+$' },
-or => [
type => { '<>' => '' },
access => { '~' => '^(read|write)' },
\'oid_parts[array_length(oid_parts,1)] = 0'
],
},{columns => [qw/mib leaf/], order_by => 'oid_parts'})
->hri->all;
debug sprintf "-> loaded %d mapped objects", scalar keys %oidmap;
return %oidmap;
}
=head2 get_oidmap_from_database( @mibs? )
=cut
sub get_oidmap_from_database {
my @mibs = @_;
debug "-> loading netdisco-mibs object cache (database)";
my %oidmap = map { ((join '::', $_->{mib}, $_->{leaf}) => $_->{oid}) }
schema('netdisco')->resultset('SNMPObject')
->search({
(scalar @mibs ? (mib => { -in => \@mibs }) : ()),
num_children => 0,
leaf => { '!~' => 'anonymous#\d+$' },
-or => [
type => { '<>' => '' },
access => { '~' => '^(read|write)' },
\'oid_parts[array_length(oid_parts,1)] = 0'
],
},{columns => [qw/mib oid leaf/], order_by => 'oid_parts'})
->hri->all;
if (not scalar @mibs) {
debug sprintf "-> loaded %d MIB objects", scalar keys %oidmap;
}
return %oidmap;
}
=head2 get_oidmap_from_mibs_files( @vendors? )
Read in netdisco-mibs translation report and make an OID -> leafname map this
is an older version of get_oidmap which uses disk file test on my laptop shows
this version is four seconds and the database takes two.
=cut
sub get_oidmap_from_mibs_files {
debug "-> loading netdisco-mibs object cache (netdisco-mibs)";
my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs'));
my $reports = catdir( $home, 'EXTRAS', 'reports' );
my @maps = map { (splitdir($_))[-1] }
grep { ! m/^(?:EXTRAS)$/ }
grep { ! m/\./ }
grep { -f }
glob (catfile( $reports, '*_oids' ));
my @report = ();
push @report, read_lines( catfile( $reports, $_ ), 'latin-1' )
for (qw(rfc_oids net-snmp_oids cisco_oids), @maps);
my %oidmap = ();
foreach my $line (@report) {
my ($oid, $qual_leaf, $rest) = split m/,/, $line;
next unless defined $oid and defined $qual_leaf;
next if exists $oidmap{$oid};
my ($mib, $leaf) = split m/::/, $qual_leaf;
$oidmap{$oid} = $leaf;
}
debug sprintf "-> loaded %d MIB objects",
scalar keys %oidmap;
return %oidmap;
}
=head2 get_mibs_for( @extra )
=cut
sub get_mibs_for {
my @extra = @_;
my @mibs = ();
my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs'));
my $cachedir = catdir( $home, 'EXTRAS', 'indexes', 'cache' );
foreach my $item (@extra) {
next unless $item;
if ($item =~ m/^[a-z0-9-]+$/) {
push @mibs, map { (split m/\s+/)[0] }
read_lines( catfile( $cachedir, $item ), 'latin-1' );
}
elsif ($item =~ m/::/) {
Module::Load::load $item;
$item .= '::MIBS';
{
no strict 'refs';
push @mibs, keys %${item};
}
}
else {
push @mibs, $item;
}
}
return @mibs;
}
=head2 get_munges( $snmpinfo )
=cut
sub get_munges {
my $snmp = shift;
my %munge_set = ();
my %munge = %{ $snmp->munge() };
my %funcs = %{ $snmp->funcs() };
my %globals = %{ $snmp->globals() };
while (my ($alias, $leaf) = each %globals) {
$munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf};
$munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias};
}
while (my ($alias, $leaf) = each %funcs) {
$munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf};
$munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias};
}
return %munge_set;
}
true;
__DATA__
agg_ports
at_paddr
bgp_peer_addr
bp_index
c_cap
c_id
c_if
c_ip
c_platform
c_port
cd11_mac
cd11_port
cd11_rateset
cd11_rxbyte
cd11_rxpkt
cd11_sigqual
cd11_sigstrength
cd11_ssid
cd11_txbyte
cd11_txpkt
cd11_txrate
cd11_uptime
class
contact
docs_if_cmts_cm_status_inet_address
dot11_cur_tx_pwr_mw
e_class
e_descr
e_fru
e_fwver
e_hwver
e_index
e_model
e_name
e_parent
e_pos
e_serial
e_swver
e_type
eigrp_peers
fw_mac
fw_port
has_topo
i_80211channel
i_alias
i_description
i_duplex
i_duplex_admin
i_err_disable_cause
i_faststart_enabled
i_ignore
i_lastchange
i_mac
i_mtu
i_name
i_speed
i_speed_admin
i_speed_raw
i_ssidbcast
i_ssidlist
i_ssidmac
i_stp_state
i_type
i_up
i_up_admin
i_vlan
i_vlan_membership
i_vlan_membership_untagged
i_vlan_type
interfaces
ip_index
ip_netmask
ipv6_addr
ipv6_addr_prefixlength
ipv6_index
ipv6_n2p_mac
ipv6_type
isis_peers
lldp_ipv6
lldp_media_cap
lldp_rem_model
lldp_rem_serial
lldp_rem_sw_rev
lldp_rem_vendor
location
model
name
ospf_peer_id
ospf_peers
peth_port_admin
peth_port_class
peth_port_ifindex
peth_port_power
peth_port_status
peth_power_status
peth_power_watts
ports
qb_fw_vlan
serial
serial1
snmpEngineID
snmpEngineTime
snmp_comm
snmp_ver
v_index
v_name
vrf_name
vtp_d_name
vtp_version

29
lib/App/Netdisco/Web/Plugin/Device/SNMP.pm
View File

@@ -41,7 +41,7 @@ ajax '/ajax/data/device/:ip/snmptree/:base' => require_login sub {
children => \0,
state => { disabled => \1 },
icon => 'icon-search',
}] unless schema(vars->{'tenant'})->resultset('DeviceSnapshot')->find($device->ip);
}] unless $device->oids->count;
return to_json [{
text => 'No MIB data. Please run `~/bin/netdisco-do loadmibs`.',
@@ -62,12 +62,19 @@ ajax '/ajax/data/snmp/typeahead' => require_login sub {
my $table = ($deviceonly ? 'DeviceBrowser' : 'SNMPObject');
my @found = schema(vars->{'tenant'})->resultset($table)
->search({ -or => [ oid => $term,
oid => { -like => ($term .'.%') },
leaf => { -ilike => ('%'. $term .'%') } ],
->search({ -or => [ 'me.oid' => $term,
'me.oid' => { -like => ($term .'.%') },
'me.leaf' => { -ilike => ('%'. $term .'%') } ],
(($deviceonly and $device) ? (ip => $device) : ()), },
{ rows => 25, columns => 'leaf', order_by => 'oid_parts' })
->get_column('leaf')->all;
{ select => [
(($deviceonly and $device) ? \q{ oid_fields.mib || '::' || me.leaf }
: \q{ me.mib || '::' || me.leaf }),
],
as => ['qleaf'],
(($deviceonly and $device) ? (join => 'oid_fields') : ()),
rows => 25, order_by => 'me.oid_parts' })
->get_column('qleaf')->all;
return to_json [] unless scalar @found;
content_type 'application/json';
@@ -87,10 +94,12 @@ ajax '/ajax/data/snmp/nodesearch' => require_login sub {
{ rows => 1, order_by => 'oid_parts' })->first;
}
else {
my ($mib, $leaf) = split m/::/, $to_match;
$found = schema(vars->{'tenant'})->resultset('SNMPObject')
->search({ -or => [ oid => $to_match,
leaf => $to_match ] },
{ rows => 1, order_by => 'oid_parts' })->first;
->search({
(($mib and $leaf) ? (-and => [mib => $mib, leaf => $leaf])
: (-or => [oid => $to_match, leaf => { -ilike => $to_match }])),
},{ rows => 1, order_by => 'oid_parts' })->first;
}
return to_json [] unless $found;
@@ -148,6 +157,8 @@ sub _get_snmp_data {
my @items = map {{
id => $_,
mib => $meta{$_}->{mib}, # accessed via node.original.mib
leaf => $meta{$_}->{leaf}, # accessed via node.original.leaf
text => ($meta{$_}->{leaf} .' ('. $meta{$_}->{oid_parts}->[-1] .')'),
($meta{$_}->{browser} ? (icon => 'icon-folder-close text-info')

34
lib/App/Netdisco/Worker/Plugin/Discover/Snapshot.pm Normal file
View File

@@ -0,0 +1,34 @@
package App::Netdisco::Worker::Plugin::Discover::Snapshot;
use Dancer ':syntax';
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Worker::Plugin;
use App::Netdisco::Transport::SNMP ();
use App::Netdisco::Util::Snapshot 'dump_cache_to_browserdata';
use Storable 'nfreeze';
use MIME::Base64 'encode_base64';
use aliased 'App::Netdisco::Worker::Status';
register_worker({ phase => 'late' }, sub {
my ($job, $workerconf) = @_;
my $device = $job->device;
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("discover failed: could not SNMP connect to $device");
return unless $device->in_storage
and not $device->oids->count and $snmp->offline;
dump_cache_to_browserdata( $device, $snmp );
my $frozen = encode_base64( nfreeze( $snmp->cache ) );
$device->update_or_create_related('snapshot', { cache => $frozen });
return Status
->info(sprintf ' [%s] discover - oids and cache stored', $device);
});
true;

477
lib/App/Netdisco/Worker/Plugin/Snapshot.pm
View File

@@ -5,16 +5,17 @@ use App::Netdisco::Worker::Plugin;
use aliased 'App::Netdisco::Worker::Status';
use App::Netdisco::Transport::SNMP;
use App::Netdisco::Util::SNMP 'sortable_oid';
use Dancer::Plugin::DBIC 'schema';
use App::Netdisco::Util::Snapshot qw/
gather_every_mib_object
dump_cache_to_browserdata
add_snmpinfo_aliases
/;
use File::Spec::Functions qw(splitdir catdir catfile);
use MIME::Base64 'encode_base64';
use File::Slurper qw(read_lines write_text);
use MIME::Base64 qw/encode_base64/;
use Storable qw/nfreeze/;
use File::Spec::Functions qw(catdir catfile);
use File::Slurper 'write_text';
use File::Path 'make_path';
use Sub::Util 'subname';
use Storable qw(dclone nfreeze);
# use DDP;
register_worker({ phase => 'check' }, sub {
return Status->error('Missing device (-d).')
@@ -26,458 +27,42 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
my ($job, $workerconf) = @_;
my $device = $job->device;
my $save_browser = $job->extra;
my $save_file = $job->port;
if (not ($device->in_storage
and not $device->is_pseudo)) {
return Status->error('Can only snapshot a real discovered device.');
}
# needed to avoid $var being returned with leafname and breaking loop checks
$SNMP::use_numeric = 1;
# might restore a cache if there's one on disk
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("snapshot failed: could not SNMP connect to $device");
my %oidmap = getoidmap($device, $snmp);
my %munges = get_munges($snmp);
# only if not pseudo device
if (not $device->is_pseudo) {
my $walk_error = walk_and_store($device, $snmp, %oidmap);
return $walk_error if $walk_error;
if ($snmp->offline) {
return Status->error('Can only snapshot a real device.');
}
# load the cache
my %cache = %{ $snmp->cache() };
gather_every_mib_object( $device, $snmp, split m/,/, ($job->extra || '') );
add_snmpinfo_aliases($snmp);
dump_cache_to_browserdata( $device, $snmp );
# finally, freeze the cache, then base64 encode, store in the DB,
# optionally store browsing data, and optionally save file.
if ($job->port) {
my $frozen = encode_base64( nfreeze( $snmp->cache ) );
if ($save_browser) {
debug "snapshot $device - cacheing snapshot for browsing";
my %seenoid = ();
if ($job->port =~ m/^(?:both|db)$/) {
debug "snapshot $device - saving snapshot to database";
$device->update_or_create_related('snapshot', { cache => $frozen });
}
my @browser = map {{
oid => $_,
oid_parts => [ grep {length} (split m/\./, $_) ],
leaf => $oidmap{$_},
munge => $munges{ $oidmap{$_} },
value => do { my $m = $oidmap{$_}; encode_base64( nfreeze( [$snmp->$m] ) ); },
}} sort {sortable_oid($a) cmp sortable_oid($b)}
grep {not $seenoid{$_}++}
grep {m/^\.1/}
map {s/^_//; $_}
keys %cache;
if ($job->port =~ m/^(?:both|file)$/) {
my $target_dir = catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots');
make_path($target_dir);
my $target_file = catfile($target_dir, $device->ip);
schema('netdisco')->txn_do(sub {
my $gone = $device->oids->delete;
debug sprintf 'snapshot %s - removed %d oids from db',
$device->ip, $gone;
$device->oids->populate(\@browser);
debug sprintf 'snapshot %s - added %d new oids to db',
$device->ip, scalar @browser;
});
}
debug "snapshot $device - cacheing snapshot bundle";
my $frozen = encode_base64( nfreeze( \%cache ) );
$device->update_or_create_related('snapshot', {cache => $frozen});
if ($save_file) {
my $target_dir = catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots');
make_path($target_dir);
my $target_file = catfile($target_dir, $device->ip);
debug "snapshot $device - saving snapshot to $target_file";
write_text($target_file, $frozen);
debug "snapshot $device - saving snapshot to $target_file";
write_text($target_file, $frozen);
}
}
return Status->done(
sprintf "Snapshot data captured from %s", $device->ip);
});
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# read in netdisco-mibs translation report and make an OID -> leafname map
sub getoidmap {
my ($device, $snmp) = @_;
debug "snapshot $device - loading netdisco-mibs object cache";
my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs'));
my $reports = catdir( $home, 'EXTRAS', 'reports' );
my @maps = map { (splitdir($_))[-1] }
grep { ! m/^(?:EXTRAS)$/ }
grep { ! m/\./ }
grep { -f }
glob (catfile( $reports, '*_oids' ));
my @report = ();
push @report, read_lines( catfile( $reports, $_ ), 'latin-1' )
for (qw(rfc_oids net-snmp_oids cisco_oids), @maps);
my %oidmap = ();
foreach my $line (@report) {
my ($oid, $qual_leaf, $rest) = split m/,/, $line;
next unless defined $oid and defined $qual_leaf;
next if exists $oidmap{$oid};
my ($mib, $leaf) = split m/::/, $qual_leaf;
$oidmap{$oid} = $leaf;
}
debug sprintf "snapshot $device - loaded %d objects from netdisco-mibs",
scalar keys %oidmap;
return %oidmap;
}
sub get_munges {
my $snmp = shift;
my %munge_set = ();
my %munge = %{ $snmp->munge() };
my %funcs = %{ $snmp->funcs() };
my %globals = %{ $snmp->globals() };
while (my ($alias, $leaf) = each %globals) {
$munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf};
$munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias};
}
while (my ($alias, $leaf) = each %funcs) {
$munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf};
$munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias};
}
return %munge_set;
}
sub walk_and_store {
my ($device, $snmp, %oidmap) = @_;
my $walk = {
%{ walker($device, $snmp, '.1.0.8802.1.1') },
%{ walker($device, $snmp, '.1.2.840.10006.300.43') },
%{ walker($device, $snmp, '.1.3.6.1') },
%{ walker($device, $snmp, '.1.3.111.2.802') },
};
# my %walk = walker($device, $snmp, '.1.3.6.1.2.1.2.2.1.6'); # 22 rows, i_mac/ifPhysAddress
# something went wrong - error
return $walk if ref {} ne ref $walk;
# take the snmpwalk of the device which is numeric (no MIB translateObj),
# resolve to MIB identifiers using netdisco-mibs, then store in SNMP::Info
# instance cache
my (%tables, %leaves, @realoids) = ((), (), ());
OID: foreach my $orig_oid (keys %$walk) {
my $oid = $orig_oid;
my $idx = '';
while (length($oid) and !exists $oidmap{$oid}) {
$oid =~ s/\.(\d+)$//;
$idx = ((defined $idx and length $idx) ? "${1}.${idx}" : $1);
}
if (exists $oidmap{$oid}) {
$idx =~ s/^\.//;
my $leaf = $oidmap{$oid};
if ($idx eq 0) {
push @realoids, $oid;
$leaves{ $leaf } = $walk->{$orig_oid};
}
else {
push @realoids, $oid if !exists $tables{ $leaf };
$tables{ $leaf }->{$idx} = $walk->{$orig_oid};
}
# debug "snapshot $device - cached $oidmap{$oid}($idx) from $orig_oid";
next OID;
}
debug "snapshot $device - missing OID $orig_oid in netdisco-mibs";
}
$snmp->_cache($_, $leaves{$_}) for keys %leaves;
$snmp->_cache($_, $tables{$_}) for keys %tables;
# add in any GLOBALS and FUNCS aliases which users have created in the
# SNMP::Info device class, with binary copy of data so that it can be frozen
my %cache = %{ $snmp->cache() };
my %funcs = %{ $snmp->funcs() };
my %globals = %{ $snmp->globals() };
while (my ($alias, $leaf) = each %globals) {
if (exists $cache{"_$leaf"} and !exists $cache{"_$alias"}) {
$snmp->_cache($alias, $cache{"_$leaf"});
}
}
while (my ($alias, $leaf) = each %funcs) {
if (exists $cache{store}->{$leaf} and !exists $cache{store}->{$alias}) {
$snmp->_cache($alias, dclone $cache{store}->{$leaf});
}
}
# now for any other SNMP::Info method in GLOBALS or FUNCS which Netdisco
# might call, but will not have data, we fake a cache entry to avoid
# throwing errors
# refresh the cache
%cache = %{ $snmp->cache() };
while (my $method = <DATA>) {
$method =~ s/\s//g;
next unless length $method and !exists $cache{"_$method"};
$snmp->_cache($method, {}) if exists $funcs{$method};
$snmp->_cache($method, '') if exists $globals{$method};
}
# put into the cache an oid ref to each leaf name
# this allows rebuild of browser data from a frozen cache
foreach my $oid (@realoids) {
my $leaf = $oidmap{$oid} or next;
$snmp->_cache($oid, $snmp->$leaf);
}
return 0;
}
# taken from SNMP::Info and adjusted to work on walks outside a single table
sub walker {
my ($device, $snmp, $base) = @_;
$base ||= '.1';
my $sess = $snmp->session();
return unless defined $sess;
my $REPEATERS = 20;
my $ver = $snmp->snmp_ver();
# debug "snapshot $device - $base translated as $qual_leaf";
my $var = SNMP::Varbind->new( [$base] );
# So devices speaking SNMP v.1 are not supposed to give out
# data from SNMP2, but most do. Net-SNMP, being very precise
# will tell you that the SNMP OID doesn't exist for the device.
# They have a flag RetryNoSuch that is used for get() operations,
# but not for getnext(). We set this flag normally, and if we're
# using V1, let's try and fetch the data even if we get one of those.
my %localstore = ();
my $errornum = 0;
my %seen = ();
my $vars = [];
my $bulkwalk_no
= $snmp->can('bulkwalk_no') ? $snmp->bulkwalk_no() : 0;
my $bulkwalk_on = defined $snmp->{BulkWalk} ? $snmp->{BulkWalk} : 1;
my $can_bulkwalk = $bulkwalk_on && !$bulkwalk_no;
my $repeaters = $snmp->{BulkRepeaters} || $REPEATERS;
my $bulkwalk = $can_bulkwalk && $ver != 1;
my $loopdetect
= defined $snmp->{LoopDetect} ? $snmp->{LoopDetect} : 1;
debug "snapshot $device - starting walk from $base";
# Use BULKWALK if we can because its faster
if ( $bulkwalk && @$vars == 0 ) {
($vars) = $sess->bulkwalk( 0, $repeaters, $var );
if ( $sess->{ErrorNum} ) {
debug "snapshot $device BULKWALK " . $sess->{ErrorStr};
debug "snapshot $device disabling BULKWALK and trying again...";
$vars = [];
$bulkwalk = 0;
$snmp->{BulkWalk} = 0;
undef $sess->{ErrorNum};
undef $sess->{ErrorStr};
}
}
while ( !$errornum ) {
if ($bulkwalk) {
$var = shift @$vars or last;
}
else {
# GETNEXT instead of BULKWALK
# debug "snapshot $device GETNEXT $var";
my @x = $sess->getnext($var);
$errornum = $sess->{ErrorNum};
}
my $iid = $var->[1];
my $val = $var->[2];
my $oid = $var->[0] . (defined $iid ? ".${iid}" : '');
# debug "snapshot $device reading $oid";
# use DDP; p $var;
unless ( defined $iid ) {
error "snapshot $device not here";
next;
}
# Check if last element, V2 devices may report ENDOFMIBVIEW even if
# instance or object doesn't exist.
if ( $val eq 'ENDOFMIBVIEW' ) {
debug "snapshot $device : ENDOFMIBVIEW";
last;
}
# Similarly for SNMPv1 - noSuchName return results in both $iid
# and $val being empty strings.
if ( $val eq '' and $iid eq '' ) {
debug "snapshot $device : v1 noSuchName (1)";
last;
}
# Another check for SNMPv1 - noSuchName return may results in an $oid
# we've already seen and $val an empty string. If we don't catch
# this here we erroneously report a loop below.
if ( defined $seen{$oid} and $seen{$oid} and $val eq '' ) {
debug "snapshot $device : v1 noSuchName (2)";
last;
}
if ($loopdetect) {
# Check to see if we've already seen this IID (looping)
if ( defined $seen{$oid} and $seen{$oid} ) {
debug "snapshot $device : looping on $oid";
shift @$vars;
$var = shift @$vars or last;
next;
}
else {
$seen{$oid}++;
}
}
if ( $val eq 'NOSUCHOBJECT' ) {
error "snapshot $device : NOSUCHOBJECT";
next;
}
if ( $val eq 'NOSUCHINSTANCE' ) {
error "snapshot $device : NOSUCHINSTANCE";
next;
}
# debug "snapshot $device - retreived $oid : $val";
$localstore{$oid} = $val;
}
debug sprintf "snapshot $device - walked %d rows from $base",
scalar keys %localstore;
return \%localstore;
}
true;
__DATA__
agg_ports
at_paddr
bgp_peer_addr
bp_index
c_cap
c_id
c_if
c_ip
c_platform
c_port
cd11_mac
cd11_port
cd11_rateset
cd11_rxbyte
cd11_rxpkt
cd11_sigqual
cd11_sigstrength
cd11_ssid
cd11_txbyte
cd11_txpkt
cd11_txrate
cd11_uptime
class
contact
docs_if_cmts_cm_status_inet_address
dot11_cur_tx_pwr_mw
e_class
e_descr
e_fru
e_fwver
e_hwver
e_index
e_model
e_name
e_parent
e_pos
e_serial
e_swver
e_type
eigrp_peers
fw_mac
fw_port
has_topo
i_80211channel
i_alias
i_description
i_duplex
i_duplex_admin
i_err_disable_cause
i_faststart_enabled
i_ignore
i_lastchange
i_mac
i_mtu
i_name
i_speed
i_speed_admin
i_speed_raw
i_ssidbcast
i_ssidlist
i_ssidmac
i_stp_state
i_type
i_up
i_up_admin
i_vlan
i_vlan_membership
i_vlan_membership_untagged
i_vlan_type
interfaces
ip_index
ip_netmask
ipv6_addr
ipv6_addr_prefixlength
ipv6_index
ipv6_n2p_mac
ipv6_type
isis_peers
lldp_ipv6
lldp_media_cap
lldp_rem_model
lldp_rem_serial
lldp_rem_sw_rev
lldp_rem_vendor
location
model
name
ospf_peer_id
ospf_peers
peth_port_admin
peth_port_class
peth_port_ifindex
peth_port_power
peth_port_status
peth_power_status
peth_power_watts
ports
qb_fw_vlan
serial
serial1
snmpEngineID
snmpEngineTime
snmp_comm
snmp_ver
v_index
v_name
vrf_name
vtp_d_name
vtp_version