#325 significant speed-up to Device > Ports tab (thx to T. Teräs)

Changes

@@ -4,6 +4,7 @@
 
   * Documentation note on OS upgrade
   * #324 use a (better) host group for internal localnet filter
+  * #325 significant speed-up to Device > Ports tab (thx to T. Teräs)
 
 2.036005 - 2017-07-05
 

lib/App/Netdisco/DB/Result/DevicePort.pm

@@ -91,26 +91,11 @@ __PACKAGE__->belongs_to( device => 'App::Netdisco::DB::Result::Device', 'ip' );
 Returns the set of C<device_port_vlan> entries associated with this Port.
 These will be both tagged and untagged. Use this relation in search conditions.
 
-See also C<all_port_vlans>.
-
 =cut
 
 __PACKAGE__->has_many( port_vlans => 'App::Netdisco::DB::Result::DevicePortVlan',
   { 'foreign.ip' => 'self.ip', 'foreign.port' => 'self.port' } );
 
-=head2 all_port_vlans
-
-Returns the set of C<device_port_vlan> entries associated with this Port.
-These will be both tagged and untagged. Use this relation when prefetching related
-C<device_port_vlan> rows.
-
-See also C<port_vlans>.
-
-=cut
-
-__PACKAGE__->has_many( all_port_vlans => 'App::Netdisco::DB::Result::DevicePortVlan',
-  { 'foreign.ip' => 'self.ip', 'foreign.port' => 'self.port' } );
-
 =head2 nodes / active_nodes / nodes_with_age / active_nodes_with_age
 
 Returns the set of Nodes whose MAC addresses are associated with this Device
@@ -241,10 +226,10 @@ __PACKAGE__->belongs_to( neighbor_alias => 'App::Netdisco::DB::Result::DeviceIp'
   sub {
       my $args = shift;
       return {
-          "$args->{foreign_alias}.ip" => { '=' =>
+          "$args->{foreign_alias}.alias" => { '=' =>
             $args->{self_resultsource}->schema->resultset('DeviceIp')
               ->search({alias => { -ident => "$args->{self_alias}.remote_ip"}},
-                       {rows => 1, columns => 'ip', alias => 'devipsub'})->as_query }
+                       {rows => 1, columns => 'alias', alias => 'devipsub'})->as_query }
       };
   },
   { join_type => 'LEFT' },
@@ -252,7 +237,7 @@ __PACKAGE__->belongs_to( neighbor_alias => 'App::Netdisco::DB::Result::DeviceIp'
 
 =head2 vlans
 
-As compared to C<port_vlans>, this relationship returns a set of VLAN
+As compared to C<port_vlans>, this relationship returns a set of Device VLAN
 row objects for the VLANs on the given port, which might be more useful if you
 want to find out details such as the VLAN name.
 
@@ -260,7 +245,7 @@ See also C<vlan_count>.
 
 =cut
 
-__PACKAGE__->many_to_many( vlans => 'all_port_vlans', 'vlan' );
+__PACKAGE__->many_to_many( vlans => 'port_vlans', 'vlan' );
 
 
 =head2 oui

lib/App/Netdisco/DB/ResultSet/Device.pm

@@ -96,7 +96,7 @@ sub search_aliases {
       {
         order_by => [qw/ me.dns me.ip /],
         join => 'device_ips',
-        distinct => 1,
+        group_by => 'me.ip',
       }
     );
 }

lib/App/Netdisco/Web/Plugin/Device/Ports.pm

@@ -113,7 +113,32 @@ get '/ajax/content/device/ports' => require_login sub {
         $set = $set->search({-or => \@combi});
     }
 
-    # get aggregate master status
+    # so far only the basic device_port data
+    # now begin to join tables depending on the selected columns/options
+
+    # get vlans on the port
+    # leave this query dormant (lazy) unless c_vmember is set
+    my $vlans = $set->search({}, {
+      select => [
+        'port',
+        { array_agg => 'port_vlans.vlan', -as => 'vlan_set'   },
+        { count     => 'port_vlans.vlan', -as => 'vlan_count' },
+      ],
+      join => 'port_vlans',
+      group_by => 'me.port',
+    });
+
+    if (param('c_vmember')) {
+        $vlans = { map {(
+          $_->port => {
+            # DBIC smart enough to work out this should be an arrayref :)
+            vlan_set   => $_->get_column('vlan_set'),
+            vlan_count => $_->get_column('vlan_count'),
+          },
+        )} $vlans->all };
+    }
+
+    # get aggregate master status (self join)
     $set = $set->search({}, {
       'join' => 'agg_master',
       '+select' => [qw/agg_master.up_admin agg_master.up/],
@@ -123,17 +148,6 @@ get '/ajax/content/device/ports' => require_login sub {
     # make sure query asks for formatted timestamps when needed
     $set = $set->with_times if param('c_lastchange');
 
-    # get vlans on the port, if there aren't too many
-    my $port_cnt = $device->ports->count() || 1;
-    my $vlan_cnt = $device->port_vlans->count() || 1;
-    my $vmember_ok =
-      (($vlan_cnt / $port_cnt) <= setting('devport_vlan_limit'));
-
-    if ($vmember_ok) {
-        $set = $set->search_rs({}, { prefetch => 'all_port_vlans' })->with_vlan_count
-          if param('c_vmember');
-    }
-
     # what kind of nodes are we interested in?
     my $nodes_name = (param('n_archived') ? 'nodes' : 'active_nodes');
     $nodes_name .= '_with_age' if param('n_age');
@@ -144,28 +158,35 @@ get '/ajax/content/device/ports' => require_login sub {
 
     if (param('c_nodes')) {
         # retrieve active/all connected nodes, if asked for
-        $set = $set->search_rs({}, { prefetch => [{$nodes_name => $ips_name}] });
+        $set = $set->search({}, { prefetch => [{$nodes_name => $ips_name}] });
-        $set = $set->search_rs({}, { order_by => ["${nodes_name}.vlan", "${nodes_name}.mac", "${ips_name}.ip"] });
+        $set = $set->search({}, { order_by => ["${nodes_name}.vlan", "${nodes_name}.mac", "${ips_name}.ip"] });
 
         # retrieve wireless SSIDs, if asked for
-        $set = $set->search_rs({}, { prefetch => [{$nodes_name => 'wireless'}] })
+        $set = $set->search({}, { prefetch => [{$nodes_name => 'wireless'}] })
           if param('n_ssid');
 
         # retrieve NetBIOS, if asked for
-        $set = $set->search_rs({}, { prefetch => [{$nodes_name => 'netbios'}] })
+        $set = $set->search({}, { prefetch => [{$nodes_name => 'netbios'}] })
           if param('n_netbios');
 
         # retrieve vendor, if asked for
-        $set = $set->search_rs({}, { prefetch => [{$nodes_name => 'oui'}] })
+        $set = $set->search({}, { prefetch => [{$nodes_name => 'oui'}] })
           if param('n_vendor');
     }
 
     # retrieve SSID, if asked for
-    $set = $set->search({}, { prefetch => 'ssid' }) if param('c_ssid');
+    $set = $set->search({}, { prefetch => 'ssid' })
+      if param('c_ssid');
 
     # retrieve neighbor devices, if asked for
-    $set = $set->search_rs({}, { prefetch => [{neighbor_alias => 'device'}] })
+    #$set = $set->search({}, { prefetch => [{neighbor_alias => 'device'}] })
-      if param('c_neighbors');
+    #  if param('c_neighbors');
+    # retrieve neighbor devices, if asked for
+    $set = $set->search({}, {
+      join => 'neighbor_alias',
+      '+select' => ['neighbor_alias.ip', 'neighbor_alias.dns'],
+      '+as'     => ['neighbor_ip', 'neighbor_dns'],
+    }) if param('c_neighbors');
 
     # sort ports (empty set would be a 'no records' msg)
     my $results = [ sort { &App::Netdisco::Util::Web::sort_port($a->port, $b->port) } $set->all ];
@@ -177,7 +198,7 @@ get '/ajax/content/device/ports' => require_login sub {
           nodes => $nodes_name,
           ips   => $ips_name,
           device => $device,
-          vmember_ok => $vmember_ok,
+          vlans  => $vlans,
         }, { layout => undef };
     }
     else {
@@ -187,6 +208,7 @@ get '/ajax/content/device/ports' => require_login sub {
           nodes => $nodes_name,
           ips   => $ips_name,
           device => $device,
+          vlans  => $vlans,
         }, { layout => undef };
     }
 };

share/views/ajax/device/ports.tt

@@ -19,11 +19,12 @@
   </thead>
   <tbody>
     [% FOREACH row IN results %]
+    [% SET portname = row.port %]
     <tr>
       <td class="nd_center-cell nd_devport-icon">
         [% IF row.up_admin != 'up' %]
         <i class="icon-remove"></i>
-        [% ELSIF row.up == 'up' AND row.stp == 'blocking' AND row.vlan_count < 2 %]
+        [% ELSIF row.up == 'up' AND row.stp == 'blocking' AND vlans.$portname.vlan_count < 2 %]
         <i class="icon-fullscreen text-info"></i>
         [% ELSIF row.has_column_loaded('is_free') AND row.is_free %]
         <i class="icon-arrow-down text-success"></i>
@@ -190,27 +191,23 @@
 
       [% IF params.c_vmember %]
       <td>
-        [% IF vmember_ok %]
+        [% IF vlans.$portname.vlan_count <= settings.devport_vlan_limit %]
-          [% IF row.vlan_count %]
           [% SET output = '' %]
-          [% SET vlanlist = [] %]
+          [% FOREACH vlan IN vlans.$portname.vlan_set.nsort %]
-          [% FOREACH vlan IN row.all_port_vlans %][% vlanlist.push(vlan.get_column('vlan')) %][% END %]
-          [% FOREACH vlan IN vlanlist.nsort %]
             [% SET output = output _
               '<a href="' _ uri_for('/search') _ '?tab=vlan&q=' _ vlan _ '">' _ vlan _ '</a>' %]
             [% SET output = output _ ', ' IF NOT loop.last %]
           [% END %]
-          [% IF row.vlan_count > 10 %] [%# TODO make this a settable variable %]
+          [% IF vlans.$portname.vlan_count > 10 %] [%# TODO make this a settable variable %]
-            [% SET output = '<div class="nd_vlan-total">(' _ row.vlan_count
+            [% SET output = '<div class="nd_vlan-total">(' _ vlans.$portname.vlan_count
               _ ')</div><span class="nd_linkcell nd_collapse-vlans">
                 <div class="nd_arrow-up-down-left icon-chevron-up icon-large"></div>Show VLANs</span>
                 <div class="nd_collapsing nd_collapse-pre-hidden">' _ output %]
             [% SET output = output _ '</div>' %]
           [% END %]
           [% output %]
-          [% END %]
         [% ELSE %]
-        <i class="icon-asterisk"></i> (too many to list)
+        <i class="icon-asterisk text-warning"></i> ([% vlans.$portname.vlan_count %] is too many to list)
         [% END %]
       </td>
       [% END %]
@@ -264,18 +261,18 @@
       [% IF params.c_nodes OR params.c_neighbors %]
       <td>
         [% IF params.c_neighbors AND (row.remote_ip OR row.is_uplink) %]
-          [% IF row.neighbor %]
+          [% IF row.get_column('neighbor_ip') %]
             <i class="icon-link[% ' text-warning' IF row.manual_topo %]"></i>
             [% IF row.remote_type AND row.remote_type.match('(?i)ip.phone') %]
               <i class="icon-phone"></i>&nbsp;
             [% ELSIF row.remote_type AND row.remote_type.match('^AP:\s') %]
               <i class="icon-signal"></i>&nbsp;
             [% END %]
-            <a href="[% uri_for('/device', self_options) %]&q=[% row.neighbor.ip | uri %]">
+            <a href="[% uri_for('/device', self_options) %]&q=[% row.get_column('neighbor_ip') | uri %]">
-                [% row.neighbor.dns.remove(settings.domain_suffix) || row.neighbor.ip | html_entity %]</a>
+                [% row.get_column('neighbor_dns').remove(settings.domain_suffix) || row.get_column('neighbor_ip') | html_entity %]</a>
             [% IF row.remote_port %]
                 -
-                <a href="[% uri_for('/device', self_options) %]&q=[% row.neighbor.ip | uri %]&f=[% row.remote_port | uri %]&prefer=port">
+                <a href="[% uri_for('/device', self_options) %]&q=[% row.get_column('neighbor_ip') | uri %]&f=[% row.remote_port | uri %]&prefer=port">
                     [% row.remote_port | html_entity %]</a>
             [% END %]
             <br/>

share/views/ajax/device/ports_csv.tt

@@ -104,18 +104,13 @@
   [% END %]
 
   [% IF params.c_vmember %]
-    [% IF row.vlan_count %]
+    [% SET portname = row.port %]
     [% SET output = '' %]
-      [% SET vlanlist = [] %]
+    [% FOREACH vlan IN vlans.$portname.vlan_set.nsort %]
-      [% FOREACH vlan IN row.all_port_vlans %][% vlanlist.push(vlan.get_column('vlan')) %][% END %]
-      [% FOREACH vlan IN vlanlist.nsort %]
       [% SET output = output _ ',' IF NOT loop.first %]
       [% SET output = output _ vlan %]
     [% END %]
     [% myport.push(output) %]
-    [% ELSE %]
-      [% myport.push('') %]
-    [% END %]
   [% END %]
 
   [% IF params.c_power %]
@@ -139,9 +134,9 @@
 
   [% IF params.c_neighbors %]
     [% IF (row.remote_ip OR row.is_uplink) %]
-      [% IF row.neighbor %]
+      [% IF row.get_column('neighbor_ip') %]
-        [% myport.push( row.neighbor.ip ) %]
+        [% myport.push( row.get_column('neighbor_ip') ) %]
-        [% myport.push( row.neighbor.dns.remove(settings.domain_suffix) ) %]
+        [% myport.push( row.get_column('neighbor_dns').remove(settings.domain_suffix) ) %]
       [% ELSIF row.remote_ip AND row.remote_port %]
         [% myport.push( row.remote_ip ) %]
         [% myport.push('') %]