make the SNMP Browser faster and colour branches with data

lib/App/Netdisco/DB.pm

@@ -11,7 +11,7 @@ __PACKAGE__->load_namespaces(
 );
 
 our # try to hide from kwalitee
-  $VERSION = 72; # schema version used for upgrades, keep as integer
+  $VERSION = 73; # schema version used for upgrades, keep as integer
 
 use Path::Class;
 use File::ShareDir 'dist_dir';

lib/App/Netdisco/DB/Result/SNMPObject.pm

@@ -21,6 +21,8 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_nullable => 1 },
   "index",
   { data_type => "text[]", is_nullable => 1, default_value => \"'{}'::text[]" },
+  "num_children",
+  { data_type => "integer", is_nullable => 0, default_value => \'0' },
 );
 __PACKAGE__->set_primary_key("oid");
 

lib/App/Netdisco/DB/Result/Virtual/FilteredSNMPObject.pm

@@ -12,11 +12,20 @@ __PACKAGE__->table("filtered_snmp_object");
 __PACKAGE__->result_source_instance->is_virtual(1);
 __PACKAGE__->result_source_instance->view_definition(<<ENDSQL
 
-    SELECT oid, oid_parts, mib, leaf, type, access, index
+    SELECT so.oid, so.oid_parts, so.mib, so.leaf, so.type, so.access, so.index, so.num_children,
-      FROM snmp_object
+           count(db.oid) AS browser
-      WHERE oid LIKE ?::text || '.%'
+      FROM snmp_object so
-        AND oid_parts[?] = ANY (?)
+
-        AND array_length(oid_parts,1) = ?
+      LEFT JOIN device_browser db ON
+           (db.ip = ? AND
+            ((so.oid = db.oid)
+              OR (array_length(db.oid_parts,1) > ?
+                  AND db.oid LIKE so.oid || '.%')))
+
+      WHERE array_length(so.oid_parts,1) = ?
+            AND so.oid LIKE ?::text || '.%'
+
+      GROUP BY so.oid, so.oid_parts, so.mib, so.leaf, so.type, so.access, so.index, so.num_children
 
 ENDSQL
 );
@@ -29,6 +38,8 @@ __PACKAGE__->add_columns(
   'type'   => { data_type => 'text' },
   'access' => { data_type => 'text' },
   'index'  => { data_type => 'text[]' },
+  'num_children' => { data_type => 'integer' },
+  'browser' => { data_type => 'integer' },
 );
 
 1;

lib/App/Netdisco/DB/Result/Virtual/OidChildren.pm

@@ -8,6 +8,10 @@ use base 'DBIx::Class::Core';
 
 __PACKAGE__->table_class('DBIx::Class::ResultSource::View');
 
+# NO LONGER USED BY NETDISCO
+# (PostgreSQL cannot handle GROUP BY using ARRAY element where
+# element ID is a bind value)
+
 __PACKAGE__->table("oid_children");
 __PACKAGE__->result_source_instance->is_virtual(1);
 __PACKAGE__->result_source_instance->view_definition(<<ENDSQL

lib/App/Netdisco/Web/Plugin/Device/SNMP.pm

@@ -121,64 +121,37 @@ sub _get_snmp_data {
     my ($ip, $base, $recurse) = @_;
     my @parts = grep {length} split m/\./, $base;
 
-    # psql cannot cope with bind params and group by array element
-    # so we build a static query instead.
-
-    my $next_part  = (scalar @parts + 1);
-    my $child_part = (scalar @parts + 2);
-    my $query = <<QUERY;
-  SELECT db.oid_parts[$next_part] AS part,
-         count(distinct(db.oid_parts[$child_part])) as children
-    FROM device_browser db
-    WHERE db.ip = ?
-          AND db.oid LIKE ? || '.%'
-    GROUP BY db.oid_parts[$next_part]
-QUERY
-    my $rs = schema('netdisco')->resultset('Virtual::GenericReport')->result_source;
-    $rs->view_definition($query);
-    $rs->remove_columns($rs->columns);
-    $rs->add_columns(qw/part children/);
-
-    my %kids = map { ($base .'.'. $_->{part}) => $_ }
-                   schema('netdisco')->resultset('Virtual::GenericReport')
-                   ->search(undef, {
-                     result_class => 'DBIx::Class::ResultClass::HashRefInflator',
-                     bind => [$ip, $base],
-                   })->hri->all;
-
-    return [{
-      text => 'No SNMP data for this device.',
-      children => \0,
-      state => { disabled => \1 },
-      icon => 'icon-search',
-    }] unless scalar keys %kids;
-
     my %meta = map { ('.'. join '.', @{$_->{oid_parts}}) => $_ }
                schema('netdisco')->resultset('Virtual::FilteredSNMPObject')
                                  ->search({}, { bind => [
+                                     $ip,
+                                     (scalar @parts + 1),
+                                     (scalar @parts + 1),
                                      $base,
-                                     (scalar @parts + 1),
-                                     [[ map {$_->{part}} values %kids ]],
-                                     (scalar @parts + 1),
                                  ] })->hri->all;
 
     my @items = map {{
         id => $_,
-        text => ($meta{$_}->{leaf} .' ('. $kids{$_}->{part} .')'),
+        text => ($meta{$_}->{leaf} .' ('. $meta{$_}->{oid_parts}->[-1] .')'),
 
-        # for nodes with only one child, recurse to prefetch...
+        ($meta{$_}->{browser} ? (icon => 'icon-folder-close text-info')
-        children => (($kids{$_}->{children} == 1)
+                              : (icon => 'icon-folder-close-alt muted')),
-          ? _get_snmp_data($ip, ("${base}.". $kids{$_}->{part}), 1)
-          : ($kids{$_}->{children} ? \1 : \0)),
 
+        (scalar @{$meta{$_}->{index}}
+          ? (icon => 'icon-th'.($meta{$_}->{browser} ? ' text-info' : ' muted')) : ()),
+
+        (($meta{$_}->{num_children} == 0 and ($meta{$_}->{type} or $meta{$_}->{oid_parts}->[-1] == 0))
+          ? (icon => 'icon-leaf'.($meta{$_}->{browser} ? ' text-info' : ' muted')) : ()),
+
+        # jstree will async call to expand these, and while it's possible
+        # for us to prefetch by calling _get_snmp_data() and passing to
+        # children, it's much slower UX. async is better for search especially
+        children => ($meta{$_}->{num_children} ? \1 : \0),
+  
         # and set the display to open to show the single child
-        state => { opened => ( ($recurse or $kids{$_}->{children} == 1)
+        state => { opened => (($meta{$_}->{num_children} == 1) ? \1 : \0 ) },
-          ? \1
-          : \0 ) },
 
-        ($kids{$_}->{children} ? () : (icon => 'icon-leaf')),
+      }} sort {$meta{$a}->{oid_parts}->[-1] <=> $meta{$b}->{oid_parts}->[-1]} keys %meta;
-        (scalar @{$meta{$_}->{index}} ? (icon => 'icon-th') : ()),
-      }} sort {$kids{$a}->{part} <=> $kids{$b}->{part}} keys %kids;
 
     return \@items;
 }

lib/App/Netdisco/Worker/Plugin/LoadMIBs.pm

@@ -19,13 +19,20 @@ register_worker({ phase => 'main' }, sub {
   my @report = read_lines(catfile($home, qw(EXTRAS reports all_oids)), 'latin-1');
 
   my @browser = ();
+  my %children = ();
+
   foreach my $line (@report) {
     my ($oid, $qual_leaf, $type, $access, $index) = split m/,/, $line;
     next unless defined $oid and defined $qual_leaf;
+
     my ($mib, $leaf) = split m/::/, $qual_leaf;
+    my @oid_parts = grep {length} (split m/\./, $oid);
+    ++$children{ join '.', '', @oid_parts[0 .. (@oid_parts - 2)] }
+      if scalar @oid_parts > 1;
+
     push @browser, {
       oid    => $oid,
-      oid_parts => [ grep {length} (split m/\./, $oid) ],
+      oid_parts => [ @oid_parts ],
       mib    => $mib,
       leaf   => $leaf,
       type   => $type,
@@ -34,6 +41,10 @@ register_worker({ phase => 'main' }, sub {
     };
   }
 
+  foreach my $row (@browser) {
+    $row->{num_children} = $children{ $row->{oid} } || 0;
+  }
+
   debug sprintf "loadmibs - loaded %d objects from netdisco-mibs",
     scalar @browser;
 

share/schema_versions/App-Netdisco-DB-72-73-PostgreSQL.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE snmp_object ADD COLUMN "num_children" integer DEFAULT 0 NOT NULL;
+
+COMMIT;