Feature to gather SNMP Walk, use as Pseudo Device, and Browse Objects

* fix anomalous name

* add gather worker

* fix encoding of binary storage

* store results back to job

* now parsing mbis report to translate

* fix the broken report parser

* rename gather to snapshot

* implement walk code copied from SNMP::Info

* can now bulkwalk and parse mibs report and store resolved walk in cache

* add func/glob aliasing broken

* better aliasing

* implement aliasing from globals and funcs

* fix regexp for matching netdisco-mibs report

* fake cache entry for all ND2 methods called, add comments

* also save to logs/snapshots/IP

* add doc for netdisco-do

* add is_pseudo column to device table

* support for loading cache for pseudo devices

* check for hrSystemUptime as well as sysUpTime for snmp connect

* display pseudo devices with yellow pill for name

* color all cells for layers for pseudo

* no need to b64 encode binary data in scalars as we b64 whole thing after

* tweaked uptime check

* store snapshot to database instead of Job

* expose snapshots in device details tab

* small ux improvements on snap download

* fixes for errors in subnet mask searching

* hide snapshot management for pseudo devices

* update to use new netdisco-mibs object cache

* update for new format oids file

* start of work on loading walk into db for browsing

* store values and meta

* add auto increment col and oid index to browser

* start web plugin for browser

* add virtual search for oid children

* have all oid in separte table (60 seconds load on my laptop)

* rename table and add relation

* store oid as int array

* fix sql for children

* make jstree start working

* working very slow tree expand

* fix to work when first displaying tree

* store both oid and oid_parts

* simplify SQL to speed up (more complicated perl)

* fix sql bug, add better index, prettify tree

* render the snmp node detail

* add node template, make scrollable, pretty print data values (insecure)

* store munge hint

* some dubious code to munge the data

* make sure to filter by IP on device_browser

* make safer the rendering of value data (but need to come back to key ordering)

* fix sorting on object values

* limit the opening of child nodes to keep response good and unclutter

* factor out the munge and make safer

* reject unknown mungers

* show the munger and option (not working) to change

* additional js for munge select

* complete custom munge

* change so that saving to database is only at CLI and on request

* hide snmp tab if no browser rows in the db

* add helpful message when no browser rows for the device

* stub handler for search and add recurse control

* working search

* minor ui fixes

* implement typeahead for leaf search

* limit rows in typeahead

* make sure device_browser is visited in delete and renumber

* add requirements for this branch

* update manifest

* make sure node search and typeahead are restricted to current device only

lib/App/Netdisco/DB/Result/Device.pm

@@ -81,6 +81,8 @@ __PACKAGE__->add_columns(
   { data_type => "timestamp", is_nullable => 1 },
   "last_arpnip",
   { data_type => "timestamp", is_nullable => 1 },
+  "is_pseudo",
+  { data_type => "boolean", is_nullable => 0, default_value => \"false" },
 );
 __PACKAGE__->set_primary_key("ip");
 
@@ -172,6 +174,14 @@ Returns the set of power modules on this Device.
 
 __PACKAGE__->has_many( power_modules => 'App::Netdisco::DB::Result::DevicePower', 'ip' );
 
+=head2 oids
+
+Returns the oids walked on this Device.
+
+=cut
+
+__PACKAGE__->has_many( oids => 'App::Netdisco::DB::Result::DeviceBrowser', 'ip' );
+
 =head2 port_vlans
 
 Returns the set of VLANs known to be configured on Ports on this Device,
@@ -256,6 +266,15 @@ Returns the row from the community string table, if one exists.
 __PACKAGE__->might_have(
     community => 'App::Netdisco::DB::Result::Community', 'ip');
 
+=head2 snapshot
+
+Returns the row from the snapshot table, if one exists.
+
+=cut
+
+__PACKAGE__->might_have(
+    snapshot => 'App::Netdisco::DB::Result::DeviceSnapshot', 'ip');
+
 =head2 throughput
 
 Returns a sum of speeds on all ports on the device.
@@ -267,17 +286,6 @@ __PACKAGE__->has_one(
 
 =head1 ADDITIONAL METHODS
 
-=head2 is_pseudo
-
-Returns true if the vendor of the device is "netdisco".
-
-=cut
-
-sub is_pseudo {
-  my $device = shift;
-  return (defined $device->vendor and $device->vendor eq 'netdisco');
-}
-
 =head2 has_layer( $number )
 
 Returns true if the device provided sysServices and supports the given layer.
@@ -314,8 +322,12 @@ sub renumber {
 
   # Community is not included as SNMP::test_connection will take care of it
   foreach my $set (qw/
+    DeviceBrowser
     DeviceIp
     DeviceModule
+    DevicePower
+    DeviceSnapshot
+    DeviceVlan
     DevicePort
     DevicePortLog
     DevicePortPower
@@ -323,8 +335,6 @@ sub renumber {
     DevicePortSsid
     DevicePortVlan
     DevicePortWireless
-    DevicePower
-    DeviceVlan
   /) {
     $schema->resultset($set)
       ->search({ip => $old_ip})
@@ -353,6 +363,11 @@ sub renumber {
     ->search({dev2 => $old_ip})
     ->update({dev2 => $new_ip});
 
+  $schema->resultset('Admin')->search({
+    device => $old_ip,
+    status => { '-not_like' => 'queued-%' },
+  })->delete;
+
   $device->update({
     ip  => $new_ip,
     dns => hostname_from_ip($new_ip),

lib/App/Netdisco/DB/Result/DeviceBrowser.pm

@@ -0,0 +1,50 @@
+use utf8;
+package App::Netdisco::DB::Result::DeviceBrowser;
+
+use strict;
+use warnings;
+
+use base 'App::Netdisco::DB::Result';
+__PACKAGE__->table("device_browser");
+__PACKAGE__->add_columns(
+  "ip",
+  { data_type => "inet", is_nullable => 0 },
+  "oid",
+  { data_type => "text", is_nullable => 0 },
+  "oid_parts",
+  { data_type => "integer[]", is_nullable => 0 },
+  "leaf",
+  { data_type => "text", is_nullable => 0 },
+  "munge",
+  { data_type => "text", is_nullable => 1 },
+  "value",
+  { data_type => "text", is_nullable => 1 },
+);
+__PACKAGE__->set_primary_key("ip", "oid");
+
+=head1 RELATIONSHIPS
+
+=head2 snmp_object
+
+Returns the SNMP Object table entry to which the given row is related. The
+idea is that you always get the SNMP Object row data even if the Device
+Browser table doesn't have any walked data.
+
+However you probably want to use the C<snmp_object> method in the
+C<DeviceBrowser> ResultSet instead, so you can pass the IP address.
+
+=cut
+
+__PACKAGE__->belongs_to(
+  snmp_object => 'App::Netdisco::DB::Result::SNMPObject',
+  sub {
+    my $args = shift;
+    return {
+        "$args->{self_alias}.oid" => { -ident => "$args->{foreign_alias}.oid" },
+        "$args->{self_alias}.ip" => { '=' => \'?' },
+    };
+  },
+  { join_type => 'RIGHT' }
+);
+
+1;

lib/App/Netdisco/DB/Result/DeviceSnapshot.pm

@@ -0,0 +1,27 @@
+use utf8;
+package App::Netdisco::DB::Result::DeviceSnapshot;
+
+use strict;
+use warnings;
+
+use base 'App::Netdisco::DB::Result';
+__PACKAGE__->table("device_snapshot");
+__PACKAGE__->add_columns(
+  "ip",
+  { data_type => "inet", is_nullable => 0 },
+  "cache",
+  { data_type => "text", is_nullable => 1 },
+);
+__PACKAGE__->set_primary_key("ip");
+
+=head1 RELATIONSHIPS
+
+=head2 device
+
+Returns the entry from the C<device> table on which this snapshot was created.
+
+=cut
+
+__PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' );
+
+1;

lib/App/Netdisco/DB/Result/SNMPObject.pm

@@ -0,0 +1,27 @@
+use utf8;
+package App::Netdisco::DB::Result::SNMPObject;
+
+use strict;
+use warnings;
+
+use base 'App::Netdisco::DB::Result';
+__PACKAGE__->table("snmp_object");
+__PACKAGE__->add_columns(
+  "oid",
+  { data_type => "text", is_nullable => 0 },
+  "oid_parts",
+  { data_type => "integer[]", is_nullable => 0 },
+  "mib",
+  { data_type => "text", is_nullable => 0 },
+  "leaf",
+  { data_type => "text", is_nullable => 0 },
+  "type",
+  { data_type => "text", is_nullable => 1 },
+  "access",
+  { data_type => "text", is_nullable => 1 },
+  "index",
+  { data_type => "text[]", is_nullable => 1, default_value => \"'{}'::text[]" },
+);
+__PACKAGE__->set_primary_key("oid");
+
+1;

lib/App/Netdisco/DB/Result/Virtual/FilteredSNMPObject.pm

@@ -0,0 +1,34 @@
+use utf8;
+package App::Netdisco::DB::Result::Virtual::FilteredSNMPObject;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
+
+__PACKAGE__->table("filtered_snmp_object");
+__PACKAGE__->result_source_instance->is_virtual(1);
+__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
+
+    SELECT oid, oid_parts, mib, leaf, type, access, index
+      FROM snmp_object
+      WHERE oid LIKE ?::text || '.%'
+        AND oid_parts[?] = ANY (?)
+        AND array_length(oid_parts,1) = ?
+
+ENDSQL
+);
+
+__PACKAGE__->add_columns(
+  'oid'    => { data_type => 'text' },
+  'oid_parts' => { data_type => 'integer[]' },
+  'mib'    => { data_type => 'text' },
+  'leaf'   => { data_type => 'text' },
+  'type'   => { data_type => 'text' },
+  'access' => { data_type => 'text' },
+  'index'  => { data_type => 'text[]' },
+);
+
+1;

lib/App/Netdisco/DB/Result/Virtual/OidChildren.pm

@@ -0,0 +1,36 @@
+use utf8;
+package App::Netdisco::DB::Result::Virtual::OidChildren;
+
+use strict;
+use warnings;
+
+use base 'DBIx::Class::Core';
+
+__PACKAGE__->table_class('DBIx::Class::ResultSource::View');
+
+__PACKAGE__->table("oid_children");
+__PACKAGE__->result_source_instance->is_virtual(1);
+__PACKAGE__->result_source_instance->view_definition(<<ENDSQL
+
+    SELECT DISTINCT(db.oid_parts[?]) AS part, count(distinct(db2.oid_parts[?])) as children
+      FROM device_browser db
+
+      LEFT JOIN device_browser db2
+      ON (db2.oid LIKE ? || '.%'
+          AND db2.oid_parts[?] = db.oid_parts[?]
+          AND db2.ip = db.ip)
+
+      WHERE db.ip = ?
+            AND db.oid LIKE ? || '.%'
+
+      GROUP BY db.oid_parts
+
+ENDSQL
+);
+
+__PACKAGE__->add_columns(
+  'part'     => { data_type => 'integer' },
+  'children' => { data_type => 'integer' },
+);
+
+1;