diff --git a/lib/App/Netdisco/Configuration.pm b/lib/App/Netdisco/Configuration.pm index 64f297b0..0a90261b 100644 --- a/lib/App/Netdisco/Configuration.pm +++ b/lib/App/Netdisco/Configuration.pm @@ -19,6 +19,21 @@ BEGIN { } } +BEGIN { + no warnings 'redefine'; + use SNMP; + + # hardware exception on macOS at least when translateObj + # gets something like '.0.0' passed as arg + + my $orig_translate = *SNMP::translateObj{'CODE'}; + *SNMP::translateObj = sub { + my $arg = $_[0]; + return undef unless defined $arg and $arg !~ m/^[.0]+$/; + return $orig_translate->(@_); + }; +} + # set up database schema config from simple config vars if (ref {} eq ref setting('database')) { # override from env for docker diff --git a/lib/App/Netdisco/DB/Result/DeviceBrowser.pm b/lib/App/Netdisco/DB/Result/DeviceBrowser.pm index d74e7d66..9735f1c2 100644 --- a/lib/App/Netdisco/DB/Result/DeviceBrowser.pm +++ b/lib/App/Netdisco/DB/Result/DeviceBrowser.pm @@ -47,4 +47,6 @@ __PACKAGE__->belongs_to( { join_type => 'RIGHT' } ); +__PACKAGE__->belongs_to( oid_fields => 'App::Netdisco::DB::Result::SNMPObject', 'oid' ); + 1; diff --git a/lib/App/Netdisco/Transport/SNMP.pm b/lib/App/Netdisco/Transport/SNMP.pm index f5cef713..ad440a24 100644 --- a/lib/App/Netdisco/Transport/SNMP.pm +++ b/lib/App/Netdisco/Transport/SNMP.pm @@ -6,16 +6,12 @@ use Dancer::Plugin::DBIC 'schema'; use App::Netdisco::Util::SNMP 'get_communities'; use App::Netdisco::Util::Device 'get_device'; use App::Netdisco::Util::Permission 'acl_matches'; +use App::Netdisco::Util::Snapshot qw/load_cache_for_device add_snmpinfo_aliases/; use SNMP::Info; use Try::Tiny; use Module::Load (); -use Storable 'thaw'; -use File::Slurper 'read_text'; -use MIME::Base64 'decode_base64'; use Path::Class 'dir'; -use File::Path 'make_path'; -use File::Spec::Functions qw(catdir catfile); use NetAddr::IP::Lite ':lower'; use List::Util qw/pairkeys pairfirst/; @@ -64,12 +60,6 @@ sub reader_for { my ($class, $ip, $useclass) = @_; my $device = get_device($ip) or return undef; - my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip ); - if ($device->in_storage and $device->is_pseudo and ! -f $pseudo_cache) { - error sprintf 'transport error - cannot act on pseudo-device [%s] without offline cache', $device->ip; - return undef; - } - my $readers = $class->instance->readers or return undef; return $readers->{$device->ip} if exists $readers->{$device->ip}; @@ -171,13 +161,13 @@ sub _snmp_connect_generic { } # support for offline cache - my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip ); - if (-f $pseudo_cache and ($device->is_pseudo or ! $device->in_storage)) { - $snmp_args{Cache} = thaw( decode_base64( read_text($pseudo_cache) ) ); - $snmp_args{Offline} = 1; - # support pseudo/offline device renumber and also pseudo device autovivification - $device->set_column(is_pseudo => \'true') if ! $device->is_pseudo; - debug sprintf 'snmp transport running in offline mode for: [%s]', $device->ip; + my $cache = load_cache_for_device($device); + if (scalar keys %$cache) { + $snmp_args{Cache} = $cache; + $snmp_args{Offline} = 1; + # support pseudo/offline device renumber and also pseudo device autovivification + $device->set_column(is_pseudo => \'true') if not $device->is_pseudo; + debug sprintf 'snmp transport running in offline mode for: [%s]', $device->ip; } # any net-snmp options to add or override @@ -229,6 +219,7 @@ sub _snmp_connect_generic { my $class = $info->device_type; return $class->new( %snmp_args, Version => $ver, + ($info->offline ? (Cache => $info->cache) : ()), _mk_info_commargs($comm), ); } @@ -298,6 +289,7 @@ sub _try_connect { Module::Load::load $class; $info = $class->new(%$snmp_args, %comm_args); + add_snmpinfo_aliases($info) if $info->offline; } } catch { @@ -388,7 +380,7 @@ sub _build_mibdirs { sub _get_mibdirs_content { my $home = shift; - my @list = map {s|$home/||; $_} grep {m/[a-z0-9]/} grep {-d} glob("$home/*"); + my @list = map {s|$home/||; $_} grep { m|/[a-z0-9-]+$| } grep {-d} glob("$home/*"); return \@list; } diff --git a/lib/App/Netdisco/Util/Device.pm b/lib/App/Netdisco/Util/Device.pm index 61280290..143101e4 100644 --- a/lib/App/Netdisco/Util/Device.pm +++ b/lib/App/Netdisco/Util/Device.pm @@ -166,9 +166,8 @@ sub is_discoverable { $remote_type ||= ''; $remote_cap ||= []; - my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip ); return _bail_msg("is_discoverable: $device is pseudo-device without offline cache") - if $device->is_pseudo and ! -f $pseudo_cache; + if $device->is_pseudo and not $device->oids->count; return _bail_msg("is_discoverable: $device matches wap_platforms but discover_waps is not enabled") if ((not setting('discover_waps')) and diff --git a/lib/App/Netdisco/Util/SNMP.pm b/lib/App/Netdisco/Util/SNMP.pm index 0de377b4..2c6a8393 100644 --- a/lib/App/Netdisco/Util/SNMP.pm +++ b/lib/App/Netdisco/Util/SNMP.pm @@ -3,8 +3,10 @@ package App::Netdisco::Util::SNMP; use Dancer qw/:syntax :script !to_json !from_json/; use App::Netdisco::Util::DeviceAuth 'get_external_credentials'; -use MIME::Base64 'decode_base64'; -use Storable 'thaw'; +use File::Spec::Functions qw/splitdir catdir catfile/; +use MIME::Base64 qw/decode_base64/; +use Storable qw/thaw/; +use SNMP::Info; use JSON::PP; use base 'Exporter'; @@ -12,9 +14,9 @@ our @EXPORT = (); our @EXPORT_OK = qw/ get_communities snmp_comm_reindex - sortable_oid - decode_and_munge %ALL_MUNGERS + decode_and_munge + sortable_oid /; our %EXPORT_TAGS = (all => \@EXPORT_OK); @@ -31,24 +33,6 @@ subroutines. =head1 EXPORT_OK -=head2 sortable_oid( $oid, $seglen? ) - -Take an OID and return a version of it which is sortable using C -operator. Works by zero-padding the numeric parts all to be length -C<< $seglen >>, which defaults to 6. - -=cut - -# take oid and make comparable -sub sortable_oid { - my ($oid, $seglen) = @_; - $seglen ||= 6; - return $oid if $oid !~ m/^[0-9.]+$/; - $oid =~ s/^(\.)//; my $leading = $1; - $oid = join '.', map { sprintf("\%0${seglen}d", $_) } (split m/\./, $oid); - return (($leading || '') . $oid); -} - =head2 get_communities( $device, $mode ) Takes the current C setting and pushes onto the front of the list @@ -247,4 +231,22 @@ sub decode_and_munge { } +=head2 sortable_oid( $oid, $seglen? ) + +Take an OID and return a version of it which is sortable using C +operator. Works by zero-padding the numeric parts all to be length +C<< $seglen >>, which defaults to 6. + +=cut + +# take oid and make comparable +sub sortable_oid { + my ($oid, $seglen) = @_; + $seglen ||= 6; + return $oid if $oid !~ m/^[0-9.]+$/; + $oid =~ s/^(\.)//; my $leading = $1; + $oid = join '.', map { sprintf("\%0${seglen}d", $_) } (split m/\./, $oid); + return (($leading || '') . $oid); +} + true; diff --git a/lib/App/Netdisco/Util/Snapshot.pm b/lib/App/Netdisco/Util/Snapshot.pm new file mode 100644 index 00000000..1a4d5008 --- /dev/null +++ b/lib/App/Netdisco/Util/Snapshot.pm @@ -0,0 +1,622 @@ +package App::Netdisco::Util::Snapshot; + +use Dancer qw/:syntax :script !to_json !from_json/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::SNMP 'sortable_oid'; + +use File::Spec::Functions qw/splitdir catdir catfile/; +use MIME::Base64 qw/encode_base64 decode_base64/; +use File::Slurper qw/read_lines read_text/; +use Sub::Util 'subname'; +use Storable qw/dclone nfreeze thaw/; +use Scalar::Util 'blessed'; +use Module::Load (); +use SNMP::Info; + +use base 'Exporter'; +our @EXPORT = (); +our @EXPORT_OK = qw/ + load_cache_for_device + snmpwalk_to_cache + + gather_every_mib_object + dump_cache_to_browserdata + add_snmpinfo_aliases + + get_oidmap_from_database + get_oidmap_from_mibs_files + get_mibs_for + get_munges +/; +our %EXPORT_TAGS = (all => \@EXPORT_OK); + +=head1 NAME + +App::Netdisco::Util::Snapshot + +=head1 DESCRIPTION + +Helper functions for L instances. + +There are no default exports, however the C<:all> tag will export all +subroutines. + +=head1 EXPORT_OK + +=head2 load_cache_for_device( $device ) + +Tries to find a device cache in database or on disk, or build one from +a net-snmp snmpwalk on disk. Returns a cache. + +=cut + +sub load_cache_for_device { + my $device = shift; + return {} unless ($device->is_pseudo or not $device->in_storage); + + # ideally we have a cache in the db + if ($device->is_pseudo and my $snapshot = $device->snapshot) { + return thaw( decode_base64( $snapshot->cache ) ); + } + + # or we have a file on disk - could be cache or walk + my $pseudo_cache = catfile( catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'), $device->ip ); + if (-f $pseudo_cache and not $device->in_storage) { + my $content = read_text($pseudo_cache); + + if ($content =~ m/^\.1/) { + my %oids = (); + + # parse the snmpwalk output which looks like + # .1.0.8802.1.1.2.1.1.1.0 = INTEGER: 30 + my @lines = split /\n/, $content; + foreach my $line (@lines) { + my ($oid, $val) = $line =~ m/^(\S+) = (?:[^:]+: )?(.+)$/; + next unless $oid and $val; + + # empty string makes the capture go wonky + $val = '' if $val =~ m/^[^:]+: ?$/; + + # remove quotes from strings + $val =~ s/^"//; + $val =~ s/"$//; + + $oids{$oid} = $val; + } + + return snmpwalk_to_cache(%oids); + } + else { + my $cache = thaw( decode_base64( $content ) ); + return add_snmpinfo_aliases( $cache ); + } + + # there is a late phase discover worker to generate the oids + # and also to save the cache into the database, because we want + # to wait for device-specific SNMP::Info class and all its methods. + } + + return {}; +} + +=head2 snmpwalk_to_cache ( %oids ) + +Take the snmpwalk of the device which is numeric (no MIB translateObj), +resolve to MIB identifiers using netdisco-mibs data, then return as an +SNMP::Info instance cache. + +=cut + +sub snmpwalk_to_cache { + my %oids = @_; + return () unless scalar keys %oids; + + my %oidmap = reverse get_oidmap_from_database(); + my %leaves = (); + + OID: foreach my $orig_oid (keys %oids) { + my $oid = $orig_oid; + my $idx = ''; + + while (length($oid) and !exists $oidmap{$oid}) { + $oid =~ s/\.(\d+)$//; + $idx = ((defined $idx and length $idx) ? "${1}.${idx}" : $1); + } + + if (exists $oidmap{$oid}) { + $idx =~ s/^\.//; + my $qleaf = $oidmap{$oid}; + my $key = $oid .'~~'. $qleaf; + + if ($idx eq 0) { + $leaves{$key} = $oids{$orig_oid}; + } + else { + # on rare occasions a vendor returns .0 and .something + delete $leaves{$key} + if defined $leaves{$key} and ref q{} eq $leaves{$key}; + $leaves{$key}->{$idx} = $oids{$orig_oid}; + } + + # debug "snapshot $device - cached $oidmap{$oid}($idx) from $orig_oid"; + next OID; + } + + # this is not too surprising + # debug sprintf "cache builder - error: missing OID %s in netdisco-mibs", $orig_oid; + } + + my $info = SNMP::Info->new({ + Offline => 1, + Cache => {}, + AutoSpecify => 0, + Session => {}, + }); + + foreach my $attr (keys %leaves) { + my ($oid, $qleaf) = split m/~~/, $attr; + my $val = $leaves{$attr}; + + # resolve the enums if needed + my $row = schema('netdisco')->resultset('SNMPObject')->find($oid); + if ($row and $row->enum) { + my %emap = map { reverse split m/\(/ } + map { s/\)//; $_ } + @{ $row->enum }; + + if (ref q{} eq ref $val) { + $val = $emap{$val} if exists $emap{$val}; + } + elsif (ref {} eq ref $val) { + foreach my $k (keys %$val) { + $val->{$k} = $emap{ $val->{$k} } + if exists $emap{ $val->{$k} }; + } + } + } + + my $leaf = $qleaf; + $leaf =~ s/.+:://; + + my $snmpqleaf = $qleaf; + $snmpqleaf =~ s/[-:]/_/g; + + # do we need this ?? $info->_cache($oid, $leaves{$attr}); + $info->_cache($leaf, $leaves{$attr}); + $info->_cache($snmpqleaf, $leaves{$attr}); + } + + debug sprintf "snmpwalk_to_cache: cache size: %d", scalar keys %{ $info->cache }; + + # inject a basic set of SNMP::Info globals and funcs aliases + # which are needed for initial device discovery + add_snmpinfo_aliases($info); + + return $info->cache; +} + +=head2 gather_every_mib_object( $device, $snmp, @extramibs? ) + +Gathers evey MIB Object in the MIBs loaded for the device and store +to the database for SNMP browser. + +Optionally add a list of vendors, MIBs, or SNMP:Info class for extra MIB +Objects from the netdisco-mibs bundle. + +The passed SNMP::Info instance has its cache update with the data. + +=cut + +sub gather_every_mib_object { + my ($device, $snmp, @extra) = @_; + + # get MIBs loaded for device + my @mibs = keys %{ $snmp->mibs() }; + my @extra_mibs = get_mibs_for(@extra); + debug sprintf "-> covering %d MIBs", (scalar @mibs + scalar @extra_mibs); + SNMP::loadModules($_) for @extra_mibs; + + # get qualified leafs for those MIBs from snmp_object + my %oidmap = get_oidmap_from_database(@mibs, @extra_mibs); + debug sprintf "-> gathering %d MIB Objects", scalar keys %oidmap; + + foreach my $qleaf (sort {sortable_oid($oidmap{$a}) cmp sortable_oid($oidmap{$b})} keys %oidmap) { + my $leaf = $qleaf; + $leaf =~ s/.+:://; + + my $snmpqleaf = $qleaf; + $snmpqleaf =~ s/[-:]/_/g; + + # gather the leaf + $snmp->$snmpqleaf; + + # skip any leaf which did not return data from device + # this works for both funcs and globals as funcs create stub global + next unless exists $snmp->cache->{'_'. $snmpqleaf}; + + # store a short name alias which is needed for netdisco actions + $snmp->_cache($leaf, $snmp->$snmpqleaf); + } +} + +=head2 dump_cache_to_browserdata( $device, $snmp ) + +Dumps any valid MIB leaf from the passed SNMP::Info instance's cache into +the Netdisco database SNMP Browser table. + +Ideally the leafs are fully qualified, but if not then a best effort will +be made to find their correct MIB. + +=cut + +sub dump_cache_to_browserdata { + my ($device, $snmp) = @_; + + my %qoidmap = get_oidmap_from_database(); + my %oidmap = get_leaf_to_qleaf_map(); + my %munges = get_munges($snmp); + + my $cache = $snmp->cache; + my %oids = (); + + foreach my $key (keys %$cache) { + next unless $key and $key =~ m/^_/; + + my $snmpqleaf = $key; + $snmpqleaf =~ s/^_//; + + my $qleaf = $snmpqleaf; + $qleaf =~ s/__/::/; + $qleaf =~ s/_/-/g; + + my $leaf = $qleaf; + $leaf =~ s/.+:://; + + next unless exists $qoidmap{$qleaf} + or (exists $oidmap{$leaf} and exists $qoidmap{ $oidmap{$leaf} }); + + my $oid = exists $qoidmap{$qleaf} ? $qoidmap{$qleaf} : $qoidmap{ $oidmap{$leaf} }; + my $data = exists $cache->{'store'}{$snmpqleaf} ? $cache->{'store'}{$snmpqleaf} + : $cache->{$key}; + next unless defined $data; + + push @{ $oids{$oid} }, { + oid => $oid, + oid_parts => [ grep {length} (split m/\./, $oid) ], + leaf => $leaf, + qleaf => $qleaf, + munge => ($munges{$snmpqleaf} || $munges{$leaf}), + value => encode_base64( nfreeze( [$data] ) ), + }; + } + + %oids = map { ($_ => [sort {length($b->{qleaf}) <=> length($a->{qleaf})} @{ $oids{$_} }]) } + keys %oids; + + schema('netdisco')->txn_do(sub { + my $gone = $device->oids->delete; + debug sprintf '-> removed %d oids from db', $gone; + $device->oids->populate([ sort {sortable_oid($a->{oid}) cmp sortable_oid($b->{oid})} + map { delete $_->{qleaf}; $_ } + map { $oids{$_}->[0] } keys %oids ]); + debug sprintf '-> added %d new oids to db', scalar keys %oids; + }); +} + +=head2 add_snmpinfo_aliases( $snmp_info_instance | $snmp_info_cache ) + +Add in any GLOBALS and FUNCS aliases from the SNMP::Info device class +or else a set of defaults that allow device discovery. Returns the cache. + +=cut + +sub add_snmpinfo_aliases { + my $info = shift or return {}; + + if (not blessed $info) { + $info = SNMP::Info->new({ + Offline => 1, + Cache => $info, + AutoSpecify => 0, + Session => {}, + }); + } + + my %globals = %{ $info->globals }; + my %funcs = %{ $info->funcs }; + + while (my ($alias, $leaf) = each %globals) { + next if $leaf =~ m/\.\d+$/; + $info->_cache($alias, $info->$leaf) if $info->$leaf; + } + + while (my ($alias, $leaf) = each %funcs) { + $info->_cache($alias, dclone $info->$leaf) if ref q{} ne ref $info->$leaf; + } + + # SNMP::Info::Layer3 has some weird aliases we can fix here + $info->_cache('serial1', $info->chassisId->{''}) if ref {} eq ref $info->chassisId; + $info->_cache('router_ip', $info->ospfRouterId->{''}) if ref {} eq ref $info->ospfRouterId; + $info->_cache('bgp_id', $info->bgpIdentifier->{''}) if ref {} eq ref $info->bgpIdentifier; + $info->_cache('bgp_local_as', $info->bgpLocalAs->{''}) if ref {} eq ref $info->bgpLocalAs; + $info->_cache('sysUpTime', $info->sysUpTimeInstance->{''}) if ref {} eq ref $info->sysUpTimeInstance + and not $info->sysUpTime; + $info->_cache('mac', $info->ifPhysAddress->{1}) if ref {} eq ref $info->ifPhysAddress; + + # now for any other SNMP::Info method in GLOBALS or FUNCS which Netdisco + # might call, but will not have data, we fake a cache entry to avoid + # throwing errors + + while (my $method = ) { + $method =~ s/\s//g; + next unless length $method and not $info->$method; + + $info->_cache($method, '') if exists $globals{$method}; + $info->_cache($method, {}) if exists $funcs{$method}; + } + + debug sprintf "add_snmpinfo_aliases: cache size: %d", scalar keys %{ $info->cache }; + return $info->cache; +} + +=head2 get_leaf_to_qleaf_map( ) + +=cut + +sub get_leaf_to_qleaf_map { + debug "-> loading database leaf to qleaf map"; + + my %oidmap = map { ( $_->{leaf} => (join '::', $_->{mib}, $_->{leaf}) ) } + schema('netdisco')->resultset('SNMPObject') + ->search({ + num_children => 0, + leaf => { '!~' => 'anonymous#\d+$' }, + -or => [ + type => { '<>' => '' }, + access => { '~' => '^(read|write)' }, + \'oid_parts[array_length(oid_parts,1)] = 0' + ], + },{columns => [qw/mib leaf/], order_by => 'oid_parts'}) + ->hri->all; + + debug sprintf "-> loaded %d mapped objects", scalar keys %oidmap; + return %oidmap; +} + +=head2 get_oidmap_from_database( @mibs? ) + +=cut + +sub get_oidmap_from_database { + my @mibs = @_; + debug "-> loading netdisco-mibs object cache (database)"; + + my %oidmap = map { ((join '::', $_->{mib}, $_->{leaf}) => $_->{oid}) } + schema('netdisco')->resultset('SNMPObject') + ->search({ + (scalar @mibs ? (mib => { -in => \@mibs }) : ()), + num_children => 0, + leaf => { '!~' => 'anonymous#\d+$' }, + -or => [ + type => { '<>' => '' }, + access => { '~' => '^(read|write)' }, + \'oid_parts[array_length(oid_parts,1)] = 0' + ], + },{columns => [qw/mib oid leaf/], order_by => 'oid_parts'}) + ->hri->all; + + if (not scalar @mibs) { + debug sprintf "-> loaded %d MIB objects", scalar keys %oidmap; + } + + return %oidmap; +} + +=head2 get_oidmap_from_mibs_files( @vendors? ) + +Read in netdisco-mibs translation report and make an OID -> leafname map this +is an older version of get_oidmap which uses disk file test on my laptop shows +this version is four seconds and the database takes two. + +=cut + +sub get_oidmap_from_mibs_files { + debug "-> loading netdisco-mibs object cache (netdisco-mibs)"; + + my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs')); + my $reports = catdir( $home, 'EXTRAS', 'reports' ); + my @maps = map { (splitdir($_))[-1] } + grep { ! m/^(?:EXTRAS)$/ } + grep { ! m/\./ } + grep { -f } + glob (catfile( $reports, '*_oids' )); + + my @report = (); + push @report, read_lines( catfile( $reports, $_ ), 'latin-1' ) + for (qw(rfc_oids net-snmp_oids cisco_oids), @maps); + + my %oidmap = (); + foreach my $line (@report) { + my ($oid, $qual_leaf, $rest) = split m/,/, $line; + next unless defined $oid and defined $qual_leaf; + next if exists $oidmap{$oid}; + my ($mib, $leaf) = split m/::/, $qual_leaf; + $oidmap{$oid} = $leaf; + } + + debug sprintf "-> loaded %d MIB objects", + scalar keys %oidmap; + return %oidmap; +} + +=head2 get_mibs_for( @extra ) + +=cut + +sub get_mibs_for { + my @extra = @_; + my @mibs = (); + + my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs')); + my $cachedir = catdir( $home, 'EXTRAS', 'indexes', 'cache' ); + + foreach my $item (@extra) { + next unless $item; + if ($item =~ m/^[a-z0-9-]+$/) { + push @mibs, map { (split m/\s+/)[0] } + read_lines( catfile( $cachedir, $item ), 'latin-1' ); + } + elsif ($item =~ m/::/) { + Module::Load::load $item; + $item .= '::MIBS'; + { + no strict 'refs'; + push @mibs, keys %${item}; + } + } + else { + push @mibs, $item; + } + } + + return @mibs; +} + +=head2 get_munges( $snmpinfo ) + +=cut + +sub get_munges { + my $snmp = shift; + my %munge_set = (); + + my %munge = %{ $snmp->munge() }; + my %funcs = %{ $snmp->funcs() }; + my %globals = %{ $snmp->globals() }; + + while (my ($alias, $leaf) = each %globals) { + $munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf}; + $munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias}; + } + + while (my ($alias, $leaf) = each %funcs) { + $munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf}; + $munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias}; + } + + return %munge_set; +} + +true; + +__DATA__ +agg_ports +at_paddr +bgp_peer_addr +bp_index +c_cap +c_id +c_if +c_ip +c_platform +c_port +cd11_mac +cd11_port +cd11_rateset +cd11_rxbyte +cd11_rxpkt +cd11_sigqual +cd11_sigstrength +cd11_ssid +cd11_txbyte +cd11_txpkt +cd11_txrate +cd11_uptime +class +contact +docs_if_cmts_cm_status_inet_address +dot11_cur_tx_pwr_mw +e_class +e_descr +e_fru +e_fwver +e_hwver +e_index +e_model +e_name +e_parent +e_pos +e_serial +e_swver +e_type +eigrp_peers +fw_mac +fw_port +has_topo +i_80211channel +i_alias +i_description +i_duplex +i_duplex_admin +i_err_disable_cause +i_faststart_enabled +i_ignore +i_lastchange +i_mac +i_mtu +i_name +i_speed +i_speed_admin +i_speed_raw +i_ssidbcast +i_ssidlist +i_ssidmac +i_stp_state +i_type +i_up +i_up_admin +i_vlan +i_vlan_membership +i_vlan_membership_untagged +i_vlan_type +interfaces +ip_index +ip_netmask +ipv6_addr +ipv6_addr_prefixlength +ipv6_index +ipv6_n2p_mac +ipv6_type +isis_peers +lldp_ipv6 +lldp_media_cap +lldp_rem_model +lldp_rem_serial +lldp_rem_sw_rev +lldp_rem_vendor +location +model +name +ospf_peer_id +ospf_peers +peth_port_admin +peth_port_class +peth_port_ifindex +peth_port_power +peth_port_status +peth_power_status +peth_power_watts +ports +qb_fw_vlan +serial +serial1 +snmpEngineID +snmpEngineTime +snmp_comm +snmp_ver +v_index +v_name +vrf_name +vtp_d_name +vtp_version diff --git a/lib/App/Netdisco/Web/Plugin/Device/SNMP.pm b/lib/App/Netdisco/Web/Plugin/Device/SNMP.pm index ecadf304..c8ba6c55 100644 --- a/lib/App/Netdisco/Web/Plugin/Device/SNMP.pm +++ b/lib/App/Netdisco/Web/Plugin/Device/SNMP.pm @@ -41,7 +41,7 @@ ajax '/ajax/data/device/:ip/snmptree/:base' => require_login sub { children => \0, state => { disabled => \1 }, icon => 'icon-search', - }] unless schema(vars->{'tenant'})->resultset('DeviceSnapshot')->find($device->ip); + }] unless $device->oids->count; return to_json [{ text => 'No MIB data. Please run `~/bin/netdisco-do loadmibs`.', @@ -62,12 +62,19 @@ ajax '/ajax/data/snmp/typeahead' => require_login sub { my $table = ($deviceonly ? 'DeviceBrowser' : 'SNMPObject'); my @found = schema(vars->{'tenant'})->resultset($table) - ->search({ -or => [ oid => $term, - oid => { -like => ($term .'.%') }, - leaf => { -ilike => ('%'. $term .'%') } ], + ->search({ -or => [ 'me.oid' => $term, + 'me.oid' => { -like => ($term .'.%') }, + 'me.leaf' => { -ilike => ('%'. $term .'%') } ], (($deviceonly and $device) ? (ip => $device) : ()), }, - { rows => 25, columns => 'leaf', order_by => 'oid_parts' }) - ->get_column('leaf')->all; + { select => [ + (($deviceonly and $device) ? \q{ oid_fields.mib || '::' || me.leaf } + : \q{ me.mib || '::' || me.leaf }), + ], + as => ['qleaf'], + (($deviceonly and $device) ? (join => 'oid_fields') : ()), + rows => 25, order_by => 'me.oid_parts' }) + ->get_column('qleaf')->all; + return to_json [] unless scalar @found; content_type 'application/json'; @@ -87,10 +94,12 @@ ajax '/ajax/data/snmp/nodesearch' => require_login sub { { rows => 1, order_by => 'oid_parts' })->first; } else { + my ($mib, $leaf) = split m/::/, $to_match; $found = schema(vars->{'tenant'})->resultset('SNMPObject') - ->search({ -or => [ oid => $to_match, - leaf => $to_match ] }, - { rows => 1, order_by => 'oid_parts' })->first; + ->search({ + (($mib and $leaf) ? (-and => [mib => $mib, leaf => $leaf]) + : (-or => [oid => $to_match, leaf => { -ilike => $to_match }])), + },{ rows => 1, order_by => 'oid_parts' })->first; } return to_json [] unless $found; @@ -148,6 +157,8 @@ sub _get_snmp_data { my @items = map {{ id => $_, + mib => $meta{$_}->{mib}, # accessed via node.original.mib + leaf => $meta{$_}->{leaf}, # accessed via node.original.leaf text => ($meta{$_}->{leaf} .' ('. $meta{$_}->{oid_parts}->[-1] .')'), ($meta{$_}->{browser} ? (icon => 'icon-folder-close text-info') diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Snapshot.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Snapshot.pm new file mode 100644 index 00000000..932a1c4c --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Snapshot.pm @@ -0,0 +1,34 @@ +package App::Netdisco::Worker::Plugin::Discover::Snapshot; + +use Dancer ':syntax'; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Worker::Plugin; +use App::Netdisco::Transport::SNMP (); +use App::Netdisco::Util::Snapshot 'dump_cache_to_browserdata'; + +use Storable 'nfreeze'; +use MIME::Base64 'encode_base64'; + +use aliased 'App::Netdisco::Worker::Status'; + +register_worker({ phase => 'late' }, sub { + my ($job, $workerconf) = @_; + my $device = $job->device; + + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + return unless $device->in_storage + and not $device->oids->count and $snmp->offline; + + dump_cache_to_browserdata( $device, $snmp ); + + my $frozen = encode_base64( nfreeze( $snmp->cache ) ); + $device->update_or_create_related('snapshot', { cache => $frozen }); + + return Status + ->info(sprintf ' [%s] discover - oids and cache stored', $device); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Snapshot.pm b/lib/App/Netdisco/Worker/Plugin/Snapshot.pm index f4527623..8c799b30 100644 --- a/lib/App/Netdisco/Worker/Plugin/Snapshot.pm +++ b/lib/App/Netdisco/Worker/Plugin/Snapshot.pm @@ -5,16 +5,17 @@ use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::Transport::SNMP; -use App::Netdisco::Util::SNMP 'sortable_oid'; -use Dancer::Plugin::DBIC 'schema'; +use App::Netdisco::Util::Snapshot qw/ + gather_every_mib_object + dump_cache_to_browserdata + add_snmpinfo_aliases +/; -use File::Spec::Functions qw(splitdir catdir catfile); -use MIME::Base64 'encode_base64'; -use File::Slurper qw(read_lines write_text); +use MIME::Base64 qw/encode_base64/; +use Storable qw/nfreeze/; +use File::Spec::Functions qw(catdir catfile); +use File::Slurper 'write_text'; use File::Path 'make_path'; -use Sub::Util 'subname'; -use Storable qw(dclone nfreeze); -# use DDP; register_worker({ phase => 'check' }, sub { return Status->error('Missing device (-d).') @@ -26,458 +27,42 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my ($job, $workerconf) = @_; my $device = $job->device; - my $save_browser = $job->extra; - my $save_file = $job->port; + if (not ($device->in_storage + and not $device->is_pseudo)) { + return Status->error('Can only snapshot a real discovered device.'); + } - # needed to avoid $var being returned with leafname and breaking loop checks - $SNMP::use_numeric = 1; - - # might restore a cache if there's one on disk my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) or return Status->defer("snapshot failed: could not SNMP connect to $device"); - my %oidmap = getoidmap($device, $snmp); - my %munges = get_munges($snmp); - - # only if not pseudo device - if (not $device->is_pseudo) { - my $walk_error = walk_and_store($device, $snmp, %oidmap); - return $walk_error if $walk_error; + if ($snmp->offline) { + return Status->error('Can only snapshot a real device.'); } - # load the cache - my %cache = %{ $snmp->cache() }; + gather_every_mib_object( $device, $snmp, split m/,/, ($job->extra || '') ); + add_snmpinfo_aliases($snmp); + dump_cache_to_browserdata( $device, $snmp ); - # finally, freeze the cache, then base64 encode, store in the DB, - # optionally store browsing data, and optionally save file. + if ($job->port) { + my $frozen = encode_base64( nfreeze( $snmp->cache ) ); - if ($save_browser) { - debug "snapshot $device - cacheing snapshot for browsing"; - my %seenoid = (); + if ($job->port =~ m/^(?:both|db)$/) { + debug "snapshot $device - saving snapshot to database"; + $device->update_or_create_related('snapshot', { cache => $frozen }); + } - my @browser = map {{ - oid => $_, - oid_parts => [ grep {length} (split m/\./, $_) ], - leaf => $oidmap{$_}, - munge => $munges{ $oidmap{$_} }, - value => do { my $m = $oidmap{$_}; encode_base64( nfreeze( [$snmp->$m] ) ); }, - }} sort {sortable_oid($a) cmp sortable_oid($b)} - grep {not $seenoid{$_}++} - grep {m/^\.1/} - map {s/^_//; $_} - keys %cache; + if ($job->port =~ m/^(?:both|file)$/) { + my $target_dir = catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'); + make_path($target_dir); + my $target_file = catfile($target_dir, $device->ip); - schema('netdisco')->txn_do(sub { - my $gone = $device->oids->delete; - debug sprintf 'snapshot %s - removed %d oids from db', - $device->ip, $gone; - $device->oids->populate(\@browser); - debug sprintf 'snapshot %s - added %d new oids to db', - $device->ip, scalar @browser; - }); - } - - debug "snapshot $device - cacheing snapshot bundle"; - my $frozen = encode_base64( nfreeze( \%cache ) ); - $device->update_or_create_related('snapshot', {cache => $frozen}); - - if ($save_file) { - my $target_dir = catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'logs', 'snapshots'); - make_path($target_dir); - my $target_file = catfile($target_dir, $device->ip); - debug "snapshot $device - saving snapshot to $target_file"; - write_text($target_file, $frozen); + debug "snapshot $device - saving snapshot to $target_file"; + write_text($target_file, $frozen); + } } return Status->done( sprintf "Snapshot data captured from %s", $device->ip); }); -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# read in netdisco-mibs translation report and make an OID -> leafname map -sub getoidmap { - my ($device, $snmp) = @_; - debug "snapshot $device - loading netdisco-mibs object cache"; - - my $home = (setting('mibhome') || catdir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs')); - my $reports = catdir( $home, 'EXTRAS', 'reports' ); - my @maps = map { (splitdir($_))[-1] } - grep { ! m/^(?:EXTRAS)$/ } - grep { ! m/\./ } - grep { -f } - glob (catfile( $reports, '*_oids' )); - - my @report = (); - push @report, read_lines( catfile( $reports, $_ ), 'latin-1' ) - for (qw(rfc_oids net-snmp_oids cisco_oids), @maps); - - my %oidmap = (); - foreach my $line (@report) { - my ($oid, $qual_leaf, $rest) = split m/,/, $line; - next unless defined $oid and defined $qual_leaf; - next if exists $oidmap{$oid}; - my ($mib, $leaf) = split m/::/, $qual_leaf; - $oidmap{$oid} = $leaf; - } - - debug sprintf "snapshot $device - loaded %d objects from netdisco-mibs", - scalar keys %oidmap; - return %oidmap; -} - -sub get_munges { - my $snmp = shift; - my %munge_set = (); - - my %munge = %{ $snmp->munge() }; - my %funcs = %{ $snmp->funcs() }; - my %globals = %{ $snmp->globals() }; - - while (my ($alias, $leaf) = each %globals) { - $munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf}; - $munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias}; - } - - while (my ($alias, $leaf) = each %funcs) { - $munge_set{$leaf} = subname($munge{$leaf}) if exists $munge{$leaf}; - $munge_set{$leaf} = subname($munge{$alias}) if exists $munge{$alias}; - } - - return %munge_set; -} - -sub walk_and_store { - my ($device, $snmp, %oidmap) = @_; - - my $walk = { - %{ walker($device, $snmp, '.1.0.8802.1.1') }, - %{ walker($device, $snmp, '.1.2.840.10006.300.43') }, - %{ walker($device, $snmp, '.1.3.6.1') }, - %{ walker($device, $snmp, '.1.3.111.2.802') }, - }; - # my %walk = walker($device, $snmp, '.1.3.6.1.2.1.2.2.1.6'); # 22 rows, i_mac/ifPhysAddress - - # something went wrong - error - return $walk if ref {} ne ref $walk; - - # take the snmpwalk of the device which is numeric (no MIB translateObj), - # resolve to MIB identifiers using netdisco-mibs, then store in SNMP::Info - # instance cache - - my (%tables, %leaves, @realoids) = ((), (), ()); - OID: foreach my $orig_oid (keys %$walk) { - my $oid = $orig_oid; - my $idx = ''; - - while (length($oid) and !exists $oidmap{$oid}) { - $oid =~ s/\.(\d+)$//; - $idx = ((defined $idx and length $idx) ? "${1}.${idx}" : $1); - } - - if (exists $oidmap{$oid}) { - $idx =~ s/^\.//; - my $leaf = $oidmap{$oid}; - - if ($idx eq 0) { - push @realoids, $oid; - $leaves{ $leaf } = $walk->{$orig_oid}; - } - else { - push @realoids, $oid if !exists $tables{ $leaf }; - $tables{ $leaf }->{$idx} = $walk->{$orig_oid}; - } - - # debug "snapshot $device - cached $oidmap{$oid}($idx) from $orig_oid"; - next OID; - } - - debug "snapshot $device - missing OID $orig_oid in netdisco-mibs"; - } - - $snmp->_cache($_, $leaves{$_}) for keys %leaves; - $snmp->_cache($_, $tables{$_}) for keys %tables; - - # add in any GLOBALS and FUNCS aliases which users have created in the - # SNMP::Info device class, with binary copy of data so that it can be frozen - - my %cache = %{ $snmp->cache() }; - my %funcs = %{ $snmp->funcs() }; - my %globals = %{ $snmp->globals() }; - - while (my ($alias, $leaf) = each %globals) { - if (exists $cache{"_$leaf"} and !exists $cache{"_$alias"}) { - $snmp->_cache($alias, $cache{"_$leaf"}); - } - } - - while (my ($alias, $leaf) = each %funcs) { - if (exists $cache{store}->{$leaf} and !exists $cache{store}->{$alias}) { - $snmp->_cache($alias, dclone $cache{store}->{$leaf}); - } - } - - # now for any other SNMP::Info method in GLOBALS or FUNCS which Netdisco - # might call, but will not have data, we fake a cache entry to avoid - # throwing errors - - # refresh the cache - %cache = %{ $snmp->cache() }; - - while (my $method = ) { - $method =~ s/\s//g; - next unless length $method and !exists $cache{"_$method"}; - - $snmp->_cache($method, {}) if exists $funcs{$method}; - $snmp->_cache($method, '') if exists $globals{$method}; - } - - # put into the cache an oid ref to each leaf name - # this allows rebuild of browser data from a frozen cache - foreach my $oid (@realoids) { - my $leaf = $oidmap{$oid} or next; - $snmp->_cache($oid, $snmp->$leaf); - } - - return 0; -} - -# taken from SNMP::Info and adjusted to work on walks outside a single table -sub walker { - my ($device, $snmp, $base) = @_; - $base ||= '.1'; - - my $sess = $snmp->session(); - return unless defined $sess; - - my $REPEATERS = 20; - my $ver = $snmp->snmp_ver(); - - # debug "snapshot $device - $base translated as $qual_leaf"; - my $var = SNMP::Varbind->new( [$base] ); - - # So devices speaking SNMP v.1 are not supposed to give out - # data from SNMP2, but most do. Net-SNMP, being very precise - # will tell you that the SNMP OID doesn't exist for the device. - # They have a flag RetryNoSuch that is used for get() operations, - # but not for getnext(). We set this flag normally, and if we're - # using V1, let's try and fetch the data even if we get one of those. - - my %localstore = (); - my $errornum = 0; - my %seen = (); - - my $vars = []; - my $bulkwalk_no - = $snmp->can('bulkwalk_no') ? $snmp->bulkwalk_no() : 0; - my $bulkwalk_on = defined $snmp->{BulkWalk} ? $snmp->{BulkWalk} : 1; - my $can_bulkwalk = $bulkwalk_on && !$bulkwalk_no; - my $repeaters = $snmp->{BulkRepeaters} || $REPEATERS; - my $bulkwalk = $can_bulkwalk && $ver != 1; - my $loopdetect - = defined $snmp->{LoopDetect} ? $snmp->{LoopDetect} : 1; - - debug "snapshot $device - starting walk from $base"; - - # Use BULKWALK if we can because its faster - if ( $bulkwalk && @$vars == 0 ) { - ($vars) = $sess->bulkwalk( 0, $repeaters, $var ); - if ( $sess->{ErrorNum} ) { - debug "snapshot $device BULKWALK " . $sess->{ErrorStr}; - debug "snapshot $device disabling BULKWALK and trying again..."; - $vars = []; - $bulkwalk = 0; - $snmp->{BulkWalk} = 0; - undef $sess->{ErrorNum}; - undef $sess->{ErrorStr}; - } - } - - while ( !$errornum ) { - if ($bulkwalk) { - $var = shift @$vars or last; - } - else { - # GETNEXT instead of BULKWALK - # debug "snapshot $device GETNEXT $var"; - my @x = $sess->getnext($var); - $errornum = $sess->{ErrorNum}; - } - - my $iid = $var->[1]; - my $val = $var->[2]; - my $oid = $var->[0] . (defined $iid ? ".${iid}" : ''); - - # debug "snapshot $device reading $oid"; - # use DDP; p $var; - - unless ( defined $iid ) { - error "snapshot $device not here"; - next; - } - - # Check if last element, V2 devices may report ENDOFMIBVIEW even if - # instance or object doesn't exist. - if ( $val eq 'ENDOFMIBVIEW' ) { - debug "snapshot $device : ENDOFMIBVIEW"; - last; - } - - # Similarly for SNMPv1 - noSuchName return results in both $iid - # and $val being empty strings. - if ( $val eq '' and $iid eq '' ) { - debug "snapshot $device : v1 noSuchName (1)"; - last; - } - - # Another check for SNMPv1 - noSuchName return may results in an $oid - # we've already seen and $val an empty string. If we don't catch - # this here we erroneously report a loop below. - if ( defined $seen{$oid} and $seen{$oid} and $val eq '' ) { - debug "snapshot $device : v1 noSuchName (2)"; - last; - } - - if ($loopdetect) { - # Check to see if we've already seen this IID (looping) - if ( defined $seen{$oid} and $seen{$oid} ) { - debug "snapshot $device : looping on $oid"; - shift @$vars; - $var = shift @$vars or last; - next; - } - else { - $seen{$oid}++; - } - } - - if ( $val eq 'NOSUCHOBJECT' ) { - error "snapshot $device : NOSUCHOBJECT"; - next; - } - if ( $val eq 'NOSUCHINSTANCE' ) { - error "snapshot $device : NOSUCHINSTANCE"; - next; - } - - # debug "snapshot $device - retreived $oid : $val"; - $localstore{$oid} = $val; - } - - debug sprintf "snapshot $device - walked %d rows from $base", - scalar keys %localstore; - return \%localstore; -} - true; - -__DATA__ -agg_ports -at_paddr -bgp_peer_addr -bp_index -c_cap -c_id -c_if -c_ip -c_platform -c_port -cd11_mac -cd11_port -cd11_rateset -cd11_rxbyte -cd11_rxpkt -cd11_sigqual -cd11_sigstrength -cd11_ssid -cd11_txbyte -cd11_txpkt -cd11_txrate -cd11_uptime -class -contact -docs_if_cmts_cm_status_inet_address -dot11_cur_tx_pwr_mw -e_class -e_descr -e_fru -e_fwver -e_hwver -e_index -e_model -e_name -e_parent -e_pos -e_serial -e_swver -e_type -eigrp_peers -fw_mac -fw_port -has_topo -i_80211channel -i_alias -i_description -i_duplex -i_duplex_admin -i_err_disable_cause -i_faststart_enabled -i_ignore -i_lastchange -i_mac -i_mtu -i_name -i_speed -i_speed_admin -i_speed_raw -i_ssidbcast -i_ssidlist -i_ssidmac -i_stp_state -i_type -i_up -i_up_admin -i_vlan -i_vlan_membership -i_vlan_membership_untagged -i_vlan_type -interfaces -ip_index -ip_netmask -ipv6_addr -ipv6_addr_prefixlength -ipv6_index -ipv6_n2p_mac -ipv6_type -isis_peers -lldp_ipv6 -lldp_media_cap -lldp_rem_model -lldp_rem_serial -lldp_rem_sw_rev -lldp_rem_vendor -location -model -name -ospf_peer_id -ospf_peers -peth_port_admin -peth_port_class -peth_port_ifindex -peth_port_power -peth_port_status -peth_power_status -peth_power_watts -ports -qb_fw_vlan -serial -serial1 -snmpEngineID -snmpEngineTime -snmp_comm -snmp_ver -v_index -v_name -vrf_name -vtp_d_name -vtp_version diff --git a/share/config.yml b/share/config.yml index 8e226999..2ee4b80f 100644 --- a/share/config.yml +++ b/share/config.yml @@ -525,6 +525,7 @@ worker_plugins: - 'Discover::PortProperties::PortAccessEntity' - 'Discover::Properties' - 'Discover::Properties::Tags' + - 'Discover::Snapshot' - 'Discover::VLANs' - 'Discover::Wireless' - 'Discover::WithNodes' diff --git a/share/public/css/netdisco.css b/share/public/css/netdisco.css index b9405e96..4a15b9ac 100644 --- a/share/public/css/netdisco.css +++ b/share/public/css/netdisco.css @@ -68,8 +68,8 @@ div.content > div.tab-content table.nd_floatinghead thead { overflow: auto; } -#snmpPartialSearch { - margin-top: -3px; +.nd_snmp_search_param { + margin-top: -3px !important; } /* fake looks for form submit buttons embedded in bootstrap dropdowns */ diff --git a/share/views/ajax/device/snmp.tt b/share/views/ajax/device/snmp.tt index b6fe6404..73714ea4 100644 --- a/share/views/ajax/device/snmp.tt +++ b/share/views/ajax/device/snmp.tt @@ -5,17 +5,31 @@
- + +   + + +   - Only this device - +   + Only this device + +
@@ -74,15 +88,23 @@