Feature to gather SNMP Walk, use as Pseudo Device, and Browse Objects

* fix anomalous name

* add gather worker

* fix encoding of binary storage

* store results back to job

* now parsing mbis report to translate

* fix the broken report parser

* rename gather to snapshot

* implement walk code copied from SNMP::Info

* can now bulkwalk and parse mibs report and store resolved walk in cache

* add func/glob aliasing broken

* better aliasing

* implement aliasing from globals and funcs

* fix regexp for matching netdisco-mibs report

* fake cache entry for all ND2 methods called, add comments

* also save to logs/snapshots/IP

* add doc for netdisco-do

* add is_pseudo column to device table

* support for loading cache for pseudo devices

* check for hrSystemUptime as well as sysUpTime for snmp connect

* display pseudo devices with yellow pill for name

* color all cells for layers for pseudo

* no need to b64 encode binary data in scalars as we b64 whole thing after

* tweaked uptime check

* store snapshot to database instead of Job

* expose snapshots in device details tab

* small ux improvements on snap download

* fixes for errors in subnet mask searching

* hide snapshot management for pseudo devices

* update to use new netdisco-mibs object cache

* update for new format oids file

* start of work on loading walk into db for browsing

* store values and meta

* add auto increment col and oid index to browser

* start web plugin for browser

* add virtual search for oid children

* have all oid in separte table (60 seconds load on my laptop)

* rename table and add relation

* store oid as int array

* fix sql for children

* make jstree start working

* working very slow tree expand

* fix to work when first displaying tree

* store both oid and oid_parts

* simplify SQL to speed up (more complicated perl)

* fix sql bug, add better index, prettify tree

* render the snmp node detail

* add node template, make scrollable, pretty print data values (insecure)

* store munge hint

* some dubious code to munge the data

* make sure to filter by IP on device_browser

* make safer the rendering of value data (but need to come back to key ordering)

* fix sorting on object values

* limit the opening of child nodes to keep response good and unclutter

* factor out the munge and make safer

* reject unknown mungers

* show the munger and option (not working) to change

* additional js for munge select

* complete custom munge

* change so that saving to database is only at CLI and on request

* hide snmp tab if no browser rows in the db

* add helpful message when no browser rows for the device

* stub handler for search and add recurse control

* working search

* minor ui fixes

* implement typeahead for leaf search

* limit rows in typeahead

* make sure device_browser is visited in delete and renumber

* add requirements for this branch

* update manifest

* make sure node search and typeahead are restricted to current device only

share/config.yml

@@ -92,6 +92,7 @@ web_plugins:
   - Device::Neighbors
   - Device::Addresses
   - Device::Vlans
+  - Device::SNMP
 extra_web_plugins: []
 sidebar_defaults:
   search_node:
@@ -392,51 +393,53 @@ job_prio:
 
 worker_plugins:
   - 'Arpnip'
+  - 'Arpnip::Hooks'
   - 'Arpnip::Nodes'
   - 'Arpnip::Subnets'
-  - 'Arpnip::Hooks'
   - 'Arpwalk'
   - 'Contact'
   - 'Delete'
   - 'Discover'
   - 'Discover::CanonicalIP'
   - 'Discover::Entities'
+  - 'Discover::Hooks'
   - 'Discover::Neighbors'
-  - 'Discover::Neighbors::Routed'
   - 'Discover::Neighbors::DOCSIS'
+  - 'Discover::Neighbors::Routed'
   - 'Discover::PortPower'
   - 'Discover::PortProperties'
   - 'Discover::Properties'
   - 'Discover::VLANs'
   - 'Discover::Wireless'
   - 'Discover::WithNodes'
-  - 'Discover::Hooks'
   - 'DiscoverAll'
   - 'DumpConfig'
   - 'Expire'
   - 'ExpireNodes'
+  - 'GetAPIKey'
   - 'Graph'
   - 'Hook'
   - 'Hook::Exec'
   - 'Hook::HTTP'
+  - 'LoadMIBs'
   - 'Location'
   - 'Macsuck'
+  - 'Macsuck::Hooks'
   - 'Macsuck::Nodes'
   - 'Macsuck::WirelessNodes'
-  - 'Macsuck::Hooks'
   - 'Macwalk'
   - 'MakeRancidConf'
-  - 'NodeMonitor'
   - 'Nbtstat'
   - 'Nbtstat::Core'
   - 'Nbtwalk'
+  - 'NodeMonitor'
   - 'PortControl'
   - 'PortName'
   - 'Power'
   - 'Psql'
   - 'Renumber'
-  - 'GetAPIKey'
   - 'Show'
+  - 'Snapshot'
   - 'Stats'
   - 'Vlan'
   - 'Vlan::Core'

share/public/css/netdisco.css

@@ -61,6 +61,17 @@ div.content > div.tab-content table.nd_floatinghead thead {
   overflow-x: hidden;
 }
 
+/* jstree scrollable */
+
+.nd_scrollable {
+    height: 85vh;
+    overflow: auto;
+}
+
+#snmpPartialSearch {
+  margin-top: -3px;
+}
+
 /* fake looks for form submit buttons embedded in bootstrap dropdowns */
 .nd_btn-link {
   display: block;

share/schema_versions/App-Netdisco-DB-66-67-PostgreSQL.sql

@@ -0,0 +1,9 @@
+BEGIN;
+
+ALTER TABLE device ADD COLUMN "is_pseudo" boolean DEFAULT false;
+
+UPDATE device SET is_pseudo = false;
+
+UPDATE device SET is_pseudo = true WHERE vendor = 'netdisco';
+
+COMMIT;

share/schema_versions/App-Netdisco-DB-67-68-PostgreSQL.sql

@@ -0,0 +1,9 @@
+BEGIN;
+
+CREATE TABLE device_snapshot (
+    "ip"    "inet",
+    "cache" "text",
+    PRIMARY KEY ("ip")
+);
+
+COMMIT;

share/schema_versions/App-Netdisco-DB-68-69-PostgreSQL.sql

@@ -0,0 +1,17 @@
+BEGIN;
+
+CREATE TABLE device_browser (
+    "ip"     "inet" NOT NULL,
+    "oid"    "text" NOT NULL,
+    "oid_parts" integer[] NOT NULL,
+    "leaf"   "text" NOT NULL,
+    "munge"  "text",
+    "value"  "text",
+    PRIMARY KEY ("ip", "oid")
+);
+
+CREATE INDEX idx_device_browser_ip_leaf ON device_browser(ip, leaf);
+
+CREATE INDEX idx_device_browser_oid__pattern on device_browser (oid text_pattern_ops);
+
+COMMIT;

share/schema_versions/App-Netdisco-DB-69-70-PostgreSQL.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+CREATE TABLE snmp_object (
+    "oid"    "text" NOT NULL,
+    "oid_parts" integer[] NOT NULL,
+    "mib"    "text" NOT NULL,
+    "leaf"   "text" NOT NULL,
+    "type"   "text",
+    "access" "text",
+    "index"  text[] DEFAULT '{}',
+    PRIMARY KEY ("oid")
+);
+
+CREATE INDEX idx_snmp_object_oid__pattern on snmp_object (oid text_pattern_ops);
+
+COMMIT;

share/views/ajax/admintask/pseudodevice.tt

@@ -33,8 +33,8 @@
         <input data-form="update" name="ports" type="number" value="[% row.port_count | html_entity %]">
       </td>
       <td class="nd_center-cell">
-        <span class="badge">&nbsp;</span><span class="badge">&nbsp;</span>
-        <a class="nd_layer-three-link" href="#" rel="tooltip" data-placement="bottom" data-offset="3" data-title="Enable Arpnip"><span class="badge[% ' badge-success' IF row.layers.substr(5,1) %]">3</span></a><span class="badge">&nbsp;</span><span class="badge">&nbsp;</span><span class="badge">&nbsp;</span><span class="badge">&nbsp;</span>
+        <span class="badge[% ' badge-success' IF row.layers.substr(7,1) %]">[% row.layers.substr(7,1) ? '1' : '&nbsp;' | none %]</span><span class="badge[% ' badge-success' IF row.layers.substr(6,1) %]">[% row.layers.substr(6,1) ? '2' : '&nbsp;' | none %]</span>
+        <a class="nd_layer-three-link" href="#" rel="tooltip" data-placement="bottom" data-offset="3" data-title="Enable Arpnip"><span class="badge[% ' badge-success' IF row.layers.substr(5,1) %]">3</span></a><span class="badge[% ' badge-success' IF row.layers.substr(4,1) %]">[% row.layers.substr(4,1) ? '4' : '&nbsp;' | none %]</span><span class="badge[% ' badge-success' IF row.layers.substr(3,1) %]">[% row.layers.substr(3,1) ? '5' : '&nbsp;' | none %]</span><span class="badge[% ' badge-success' IF row.layers.substr(2,1) %]">[% row.layers.substr(2,1) ? '6' : '&nbsp;' | none %]</span><span class="badge[% ' badge-success' IF row.layers.substr(1,1) %]">[% row.layers.substr(1,1) ? '7' : '&nbsp;' | none %]</span>
         <input data-form="update" name="layers" type="hidden" value="[% row.layers | html_entity %]">
       </td>
       <td class="nd_center-cell">

share/views/ajax/device/details.tt

@@ -180,6 +180,24 @@
         <input type="hidden" data-form="nbtstat" value="[% d.ip | html_entity %]" name="device"/>
         <button class="btn btn-info btn-small nd_adminbutton" name="nbtstat">NBTstat</button>
 
+        [% IF NOT d.is_pseudo %]
+        <span class="dropdown">
+          <button class="nd_snap_btn btn [% d.has_snapshot ? 'btn-success' : 'btn-info' %] btn-small dropdown-toggle" type="button" id="snapshotmenu" data-toggle="dropdown" aria-haspopup="true" aria-expanded="true">
+            Snapshot
+            <span class="caret"></span>
+          </button>
+          <ul class="dropdown-menu" aria-labelledby="snapshotmenu">
+            <input type="hidden" data-form="snapshot_req" value="[% d.ip | html_entity %]" name="device"/>
+            <li><a href="#" class="nd_adminbutton" name="snapshot_req"><i class="icon-edit"></i> Create</a></li>
+
+            <li class='nd_snap_func [% 'disabled' UNLESS d.has_snapshot %]'><a href="[% uri_for('/ajax/content/admin/snapshot_get') | none %]?device=[% d.ip | uri %]"><i class="icon-download-alt"></i> Download</a></li>
+
+            <input type="hidden" data-form="snapshot_del" value="[% d.ip | html_entity %]" name="device"/>
+            <li class='nd_snap_func [% 'disabled' UNLESS d.has_snapshot %]'><a href="#" class="nd_adminbutton" name="snapshot_del"><i class="icon-trash"></i> Delete</a></li>
+          </ul>
+        </span>
+        [% END %]
+
         <button class="btn btn-danger btn-small pull-right"
           data-toggle="modal" data-target="#nd_devdel" type="button">Delete</button>
 

share/views/ajax/device/snmp.tt

@@ -0,0 +1,132 @@
+<div class="row">
+    <div class="span5">
+        <div id="jstree" class="nd_scrollable"></div>
+    </div>
+    <div id="snmpnodecontainer" class="span8">
+        <form id="searchTreeForm" class="form-inline col-md-4">
+            <span class="form-group">
+              <input id="treeSearchText" type="text" class="form-control" size="30" required placeholder="Search for label or OID">
+              <label class="checkbox-inline"
+                rel="tooltip" data-placement="top" data-offset="5" data-title="Anchored to the beginning">
+                <input type="checkbox" id="snmpPartialSearch" value="partial"> Partial </input>
+              </label>
+            </span>
+        <button type="submit" class="btn btn-default">Search</button>
+        </form>
+        <div id="node">
+          <table class="table table-bordered">
+              <tbody>
+                <tr>
+                  <th scope="row" class="span1">OID</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Module</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Leaf</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Type</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Munge</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Access</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Index</th>
+                  <td></td>
+                </tr>
+                <tr>
+                  <th scope="row">Value</th>
+                  <td></td>
+                </tr>
+              </tbody>    
+            </table>
+        </div>
+    </div>
+</div>
+
+<script type="text/javascript">
+  $(function () {
+    var jstree_search_callback = function(str, node) {
+      var pattern = str.toLowerCase();
+      var name = node.text.toLowerCase();
+      var oid = node.id.toLowerCase();
+
+      if (document.getElementById('snmpPartialSearch').checked) {
+        if ((name.indexOf(pattern) == 0) || (oid.indexOf(pattern) == 0)) {
+          return true;
+        }
+      }
+      else {
+        if ((name.indexOf(pattern + ' ') == 0) || (oid == pattern)) {
+          return true;
+        }
+      }
+      return false;
+    };
+
+    $('#jstree').jstree({
+      'core': {
+        'multiple' : false,
+        'themes': {
+          'name': 'proton',
+          'responsive': true
+        },
+        'data' : {
+          'url' : function (node) {
+            return ('[% uri_base | none %]/ajax/data/device/[% device %]/snmptree/'
+              + (node.id === '#' ? '.1.3.6.1' : node.id) + '?recurse=on');
+          }
+        }
+      },
+      'plugins': ['search'],
+      'search': {
+        'ajax' : {
+          'url' : '[% uri_base | none %]/ajax/data/device/[% device %]/snmpnodesearch',
+          'beforeSend' : function(jqXHR, settings) {
+            settings.url = settings.url + '&excludeself=on';
+            if (document.getElementById('snmpPartialSearch').checked) {
+              settings.url = settings.url + '&partial=on';
+            }
+            return true;
+          }
+        },
+        'search_callback' : jstree_search_callback
+      },
+    });
+    $('#snmpnodecontainer').on("change", "#munger", function(e, data) {
+      var ary = $('#jstree').jstree('get_selected');
+      $('#node').load('[% uri_base | none %]/ajax/content/device/[% device %]/snmpnode/'
+        + ary[0] + '?munge=' + $('#munger').find(":selected").text());
+    });
+    $('#jstree').on("changed.jstree", function (e, data) {
+      if (data.selected && data.selected != "#") {
+        $('#node').load('[% uri_base | none %]/ajax/content/device/[% device %]/snmpnode/' + data.selected);
+      }
+    });
+    $('#jstree').on("search.jstree", function (e, data) {
+      if (data.res.length) {
+        document.getElementById( data.res[0] + '_anchor' ).scrollIntoView();
+        $('#node').load('[% uri_base | none %]/ajax/content/device/[% device %]/snmpnode/' + data.res[0]);
+      }
+    });
+    $("#searchTreeForm").submit(function(e) {
+      $("#jstree").jstree("search", $("#treeSearchText").val());
+      e.preventDefault();
+    });
+    $('#treeSearchText').autocomplete({
+      source: uri_base + '/ajax/data/device/[% device %]/typeahead'
+      ,delay: 150
+      ,minLength: 2
+    });
+  });
+</script>

share/views/ajax/device/snmpnode.tt

@@ -0,0 +1,53 @@
+<table class="table table-bordered">
+  <tbody>
+    <tr>
+      <th scope="row" class="span1">OID</th>
+      <td>[% node.snmp_object.oid %]</td>
+    </tr>
+    <tr>
+      <th scope="row">Module</th>
+      <td>[% node.snmp_object.mib %]</td>
+    </tr>
+    <tr>
+      <th scope="row">Leaf</th>
+      <td>[% node.snmp_object.leaf %]</td>
+    </tr>
+    <tr>
+      <th scope="row">Type</th>
+      <td>[% node.snmp_object.type %]</td>
+    </tr>
+    <tr>
+      <th scope="row">Munge</th>
+      <td>
+      [% IF node.value %]
+      <select name="munger" id="munger">
+        [% UNLESS munge %]<option value="" selected disabled hidden>None applied</option>[% END %]
+        [% FOREACH m IN mungers %]
+        <option [% 'selected' IF m == munge %] value="[% m %]">[% m %]</option>
+        [% END %]
+      </select>
+      [% END %]
+      </td>
+    </tr>
+    <tr>
+      <th scope="row">Access</th>
+      <td>[% node.snmp_object.access %]</td>
+    </tr>
+    <tr>
+      <th scope="row">Index</th>
+      <td>
+      [% IF node.snmp_object.index.size > 0 %]
+        <table class="table table-condensed table-bordered">
+          [% FOREACH idx IN node.snmp_object.index %]
+          <tr><td>[% idx %]</td></tr>
+          [% END %]
+        </table>
+      [% END %]
+      </td>
+    </tr>
+    <tr>
+      <th scope="row">Value</th>
+      <td>[% IF node.value %]<pre id="snmp_node_value">[% node.value %]</pre>[% END %]</td>
+    </tr>
+  </tbody>    
+</table>

share/views/device.tt

@@ -36,10 +36,11 @@
   <div class="content">
     <ul id="nd_search-results" class="nav nav-tabs">
       [% FOREACH tab IN settings._device_tabs %]
+      [% NEXT UNLESS tab.render_if %]
       <li[% ' class="active"' IF params.tab == tab.tag %]><a id="[% tab.tag | html_entity %]_link" href="#[% tab.tag | html_entity %]_pane">[% tab.label | html_entity %]</a></li>
       [% END %]
       <span id="nd_device-name">
-        [% display_name | html_entity %]
+        [% IF is_pseudo %]<span class="badge badge-warning">[% END %][% display_name | html_entity %][% IF is_pseudo %]</span>[% END %]
         <a id="nd_csv-download" href="#" download="netdisco.csv">&nbsp;
           <i id="nd_csv-download-icon" class="text-info icon-file-text-alt icon-large"
             rel="tooltip" data-placement="left" data-offset="5" data-title="Download as CSV"></i></a>
@@ -47,6 +48,7 @@
     </ul>
     <div class="tab-content">
       [% FOREACH tab IN settings._device_tabs %]
+      [% NEXT UNLESS tab.render_if %]
       <div class="tab-pane[% ' active' IF params.tab == tab.tag %]" id="[% tab.tag | html_entity %]_pane"></div>
       [% END %]
   </div>

share/views/js/device.js

@@ -169,6 +169,11 @@
         ,success: function() {
           if (mode != 'delete') {
             toastr.info('Requested '+ mode +' for device '+ tr.data('for-device'));
+            if (mode == 'snapshot_del') {
+                $('.nd_snap_btn').toggleClass('btn-success');
+                $('.nd_snap_btn').toggleClass('btn-info');
+                $('.nd_snap_func').toggleClass('disabled');
+            }
           }
           else {
             toastr.success('Deleted device '+ tr.data('for-device'));

share/views/layouts/main.tt

@@ -32,6 +32,7 @@
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/dataTables.bootstrap.js"></script>
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/dataTables.ip-address-detect.js"></script>
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/dataTables.ip-address-sort.js"></script>
+  <script type="text/javascript" src="[% uri_base | none %]/javascripts/jstree/jstree.min.js"></script>
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/he.js"></script>
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/natural.js"></script>
   <script type="text/javascript" src="[% uri_base | none %]/javascripts/portsort.js"></script>
@@ -61,6 +62,7 @@
   <link rel="stylesheet" href="[% uri_base | none %]/css/d3-force-network-chart.css"/>
   <link rel="stylesheet" href="[% uri_base | none %]/css/netdisco.css"/>
   <link rel="stylesheet" href="[% uri_base | none %]/css/bootstrap-tree.css"/>
+  <link rel="stylesheet" href="[% uri_base | none %]/javascripts/jstree/themes/proton/style.min.css"/>
   <link rel="stylesheet" href="[% uri_base | none %]/css/daterangepicker-bs2.css"/>
   <link rel="stylesheet" href="[% uri_base | none %]/css/dataTables.bootstrap.css"/>
   <link rel="stylesheet" href="[% uri_base | none %]/css/nd_print.css" media="print"/>