custom fields on devices and ports in the web from config (#945)

* custom device field web display and edit

* make display work; relies on T::T calling dict slot or method with same syntax

* add storing port custom fields

* use resultset method instead, use cf_ prefix

* update Pg min ver for jsonb

* allow override of position and default for port custom fields

* support hidden for custom fields

* update description of Objects API class

* allow left and mid position for custom fields

* add custom fields in csv

* change port control sidebar label

* fix default missing bug on backend jobs

lib/App/Netdisco.pm

@@ -103,7 +103,7 @@ will take about 250MB including MIB files.
  root:~# useradd -m -p x -s /bin/bash netdisco
 
 Netdisco uses the PostgreSQL database server. Install PostgreSQL (at least
-version 9.4) and then change to the PostgreSQL superuser (usually
+version 9.6) and then change to the PostgreSQL superuser (usually
 C<postgres>). Create a new database and PostgreSQL user for the Netdisco
 application:
 

lib/App/Netdisco/DB.pm

@@ -11,7 +11,7 @@ __PACKAGE__->load_namespaces(
 );
 
 our # try to hide from kwalitee
-  $VERSION = 76; # schema version used for upgrades, keep as integer
+  $VERSION = 77; # schema version used for upgrades, keep as integer
 
 use Path::Class;
 use File::ShareDir 'dist_dir';

lib/App/Netdisco/DB/Result/Device.pm

@@ -85,6 +85,8 @@ __PACKAGE__->add_columns(
   { data_type => "boolean", is_nullable => 0, default_value => \"false" },
   "pae_is_enabled",
   { data_type => "boolean", is_nullable => 1 },
+  "custom_fields",
+  { data_type => "jsonb", is_nullable => 0, default_value => \"{}" },
 );
 __PACKAGE__->set_primary_key("ip");
 

lib/App/Netdisco/DB/Result/DevicePort.pm

@@ -71,6 +71,8 @@ __PACKAGE__->add_columns(
   { data_type => "integer", is_nullable => 1 },
   "lastchange",
   { data_type => "bigint", is_nullable => 1 },
+  "custom_fields",
+  { data_type => "jsonb", is_nullable => 0, default_value => \"{}" },
 );
 __PACKAGE__->set_primary_key("port", "ip");
 

lib/App/Netdisco/JobQueue/PostgreSQL.pm

@@ -332,18 +332,44 @@ sub jq_insert {
   my $happy = false;
   try {
     schema(vars->{'tenant'})->txn_do(sub {
-      schema(vars->{'tenant'})->resultset('Admin')->populate([
+      if (scalar @$jobs == 1 and defined $jobs->[0]->{device} and
-        map {{
+          scalar grep {$_ eq $jobs->[0]->{action}} @{ setting('_inline_actions') || [] }) {
-            device     => $_->{device},
+
-            device_key => $_->{device_key},
+          my $spec = $jobs->[0];
-            port       => $_->{port},
+          my $row = undef;
-            action     => $_->{action},
+
-            subaction  => ($_->{extra} || $_->{subaction}),
+          if ($spec->{port}) {
-            username   => $_->{username},
+              $row = schema(vars->{'tenant'})->resultset('DevicePort')
-            userip     => $_->{userip},
+                                             ->find($spec->{port}, $spec->{device});
-            status     => 'queued',
+          }
-        }} @$jobs
+          else {
-      ]);
+              $row = schema(vars->{'tenant'})->resultset('Device')
+                                             ->find($spec->{device});
+          }
+          die 'failed to find row for custom field update' unless $row;
+
+          $spec->{action} =~ s/^cf_//;
+          $spec->{subaction} = to_json( $spec->{subaction} );
+          $row->make_column_dirty('custom_fields');
+          $row->update({
+            custom_fields => \['jsonb_set(custom_fields, ?, ?)'
+                              => (qq{{$spec->{action}}}, $spec->{subaction}) ]
+            })->discard_changes();
+      }
+      else {
+          schema(vars->{'tenant'})->resultset('Admin')->populate([
+            map {{
+                device     => $_->{device},
+                device_key => $_->{device_key},
+                port       => $_->{port},
+                action     => $_->{action},
+                subaction  => ($_->{extra} || $_->{subaction}),
+                username   => $_->{username},
+                userip     => $_->{userip},
+                status     => 'queued',
+            }} @$jobs
+          ]);
+      }
     });
     $happy = true;
   }

lib/App/Netdisco/Web.pm

@@ -111,6 +111,7 @@ use App::Netdisco::Web::TypeAhead;
 use App::Netdisco::Web::PortControl;
 use App::Netdisco::Web::Statistics;
 use App::Netdisco::Web::Password;
+use App::Netdisco::Web::CustomFields;
 use App::Netdisco::Web::GenericReport;
 
 sub _load_web_plugins {
@@ -391,7 +392,7 @@ $swagger_doc->{tags} = [
   {name => 'Search',
     description => 'Search Operations'},
   {name => 'Objects',
-    description => 'Retrieve Device, Port, and associated Node Data'},
+    description => 'Device, Port, and associated Node Data'},
   {name => 'Reports',
     description => 'Canned and Custom Reports'},
 ];

lib/App/Netdisco/Web/CustomFields.pm

@@ -0,0 +1,86 @@
+package App::Netdisco::Web::CustomFields;
+
+use Dancer ':syntax';
+use Dancer::Plugin::DBIC;
+
+use App::Netdisco::DB::ResultSet::Device;
+use App::Netdisco::DB::ResultSet::DevicePort;
+
+use App::Netdisco::Web::Plugin;
+
+my @inline_device_actions = ();
+my @inline_device_port_actions = ();
+
+foreach my $config (@{ setting('custom_fields')->{'device'} || [] }) {
+
+  if (! $config->{'name'}) {
+      error 'custom_field missing name';
+      next;
+  }
+
+  register_device_details({
+    %{ $config },
+    field => ('cf_' . $config->{'name'}),
+    label => ($config->{'label'} || ucfirst $config->{'name'}),
+  }) unless $config->{'hidden'};
+
+  push @inline_device_actions, $config->{'name'};
+
+}
+
+foreach my $config (@{ setting('custom_fields')->{'device_port'} || [] }) {
+
+  if (! $config->{'name'}) {
+      error 'custom_field missing name';
+      next;
+  }
+
+  register_device_port_column({
+    position => 'right', # or "mid" or "right"
+    default  => undef,   # or undef
+    %{ $config },
+    field => ('cf_' . $config->{'name'}),
+    label => ($config->{'label'} || ucfirst $config->{'name'}),
+  }) unless $config->{'hidden'};
+
+  push @inline_device_port_actions, $config->{'name'};
+
+}
+
+{
+  package App::Netdisco::DB::ResultSet::Device;
+
+  sub with_custom_fields {
+    my ($rs, $cond, $attrs) = @_;
+
+    return $rs
+      ->search_rs($cond, $attrs)
+      ->search({},
+        { '+columns' => {
+            map {( ('cf_'. $_) => \[ 'me.custom_fields ->> ?' => $_ ] )}
+                @inline_device_actions
+        }});
+  }
+}
+
+{
+  package App::Netdisco::DB::ResultSet::DevicePort;
+
+  sub with_custom_fields {
+    my ($rs, $cond, $attrs) = @_;
+
+    return $rs
+      ->search_rs($cond, $attrs)
+      ->search({},
+        { '+columns' => {
+            map {( ('cf_'. $_) => \[ 'me.custom_fields ->> ?' => $_ ] )}
+                @inline_device_port_actions
+        }});
+  }
+}
+
+set('_inline_actions' => [
+  map {'cf_' . $_} (@inline_device_actions, @inline_device_port_actions)
+]);
+
+true;

lib/App/Netdisco/Web/Plugin/Device/Details.pm

@@ -23,8 +23,7 @@ ajax '/ajax/content/device/details' => require_login sub {
             '+as' => ['has_snapshot'],
             join => 'snapshot',
           },
-        )->with_times()
+        )->with_times->with_custom_fields->hri->all;
-        ->hri->all;
 
     my @power
         = schema(vars->{'tenant'})->resultset('DevicePower')

lib/App/Netdisco/Web/Plugin/Device/Ports.pm

@@ -20,7 +20,7 @@ get '/ajax/content/device/ports' => require_login sub {
 
     my $device = schema(vars->{'tenant'})->resultset('Device')
       ->search_for_device($q) or send_error('Bad device', 400);
-    my $set = $device->ports->with_properties;
+    my $set = $device->ports->with_properties->with_custom_fields;
 
     # refine by ports if requested
     my $f = param('f');

lib/App/Netdisco/Web/PortControl.pm

@@ -24,7 +24,7 @@ ajax '/ajax/portcontrol' => require_any_role [qw(admin port_control)] => sub {
       'c_power'  => 'power',
     );
 
-    my $action = $action_map{ param('field') };
+    my $action = ($action_map{ param('field') } || param('field') || '');
     my $subaction = ($action =~ m/^(?:power|portcontrol)/
       ? (param('action') ."-other")
       : param('value'));
@@ -34,6 +34,7 @@ ajax '/ajax/portcontrol' => require_any_role [qw(admin port_control)] => sub {
           my $act = "$action $subaction";
           $act =~ s/-other$//;
           $act =~ s/^portcontrol/port/;
+          $act =~ s/^device_port_custom_field_/custom_field: /;
 
           schema(vars->{'tenant'})->resultset('DevicePortLog')->create({
             ip => param('device'),

share/config.yml

@@ -112,7 +112,7 @@ sidebar_defaults:
   search_device:
     matchall:    { default: checked }
   device_ports:
-    c_admin:       { label: 'Port Controls', default: null, idx: 0 }
+    c_admin:       { label: 'Port Control and Editing', default: null, idx: 0 }
     c_port:        { label: 'Port', default: checked, idx: 1 }
     c_descr:       { label: 'Description', default: null, idx: 2 }
     c_comment:     { label: 'Last Comment', default: null, idx: 3 }
@@ -275,6 +275,9 @@ community_rw: []
 device_auth: []
 use_legacy_rancidexport: false
 use_legacy_sshcollector: false
+custom_fields:
+  device: []
+  device_port: []
 get_credentials: ""
 bulkwalk_off: false
 bulkwalk_no: []

share/schema_versions/App-Netdisco-DB-76-77-PostgreSQL.sql

@@ -0,0 +1,11 @@
+BEGIN;
+
+ALTER TABLE device ADD COLUMN "custom_fields" jsonb DEFAULT '{}';
+
+UPDATE device SET custom_fields = '{}';
+
+ALTER TABLE device_port ADD COLUMN "custom_fields" jsonb DEFAULT '{}';
+
+UPDATE device_port SET custom_fields = '{}';
+
+COMMIT;

share/views/ajax/device/details.tt

@@ -79,14 +79,25 @@
     <tr>
       <td>
           [% config.label | html_entity %]
-      </td>
+          [% IF config.editable AND user_can_port_control %]
-      <td>
+          <i class="icon-edit nd_edit-icon nd_device-details-edit"></i>
-          [% TRY %]
-          [% INCLUDE "plugin/${config.name}/device_details.tt" %]
-          [% CATCH %]
-          <!-- dummy content required by Template Toolkit TRY -->
           [% END %]
       </td>
+      [% TRY %]
+        <td>
+        [% INCLUDE "plugin/${config.name}/device_details.tt" %]
+        </td>
+      [% CATCH %]
+        [% CLEAR %]
+        [% IF config.editable AND user_can_port_control %]
+          <td class="nd_editable-cell" contenteditable="true"
+            data-field="[% config.field | html_entity %]" data-for-device="[% d.ip | html_entity %]">
+        [% ELSE %]
+          <td>
+        [% END %]
+          [% d.${config.field} | html_entity %]
+          </td>
+      [% END %]
     </tr>
     [% END %]
     [% END %]

share/views/ajax/device/ports.tt

@@ -60,13 +60,26 @@
 
       [% FOREACH config IN settings._extra_device_port_cols %]
         [% NEXT UNLESS config.position == 'left' AND params.${config.name} %]
-        <td>
+
-          [% TRY %]
+        [% TRY %]
+          <td>
           [% INCLUDE "plugin/${config.name}/device_port_column.tt" %]
-          [% CATCH %]
+          </td>
-          <!-- dummy content required by Template Toolkit TRY -->
+        [% CATCH %]
+          [% CLEAR %]
+          [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %]
+            <td nowrap class="nd_editable-cell" contenteditable="true"
+                data-field="[% config.field | html_entity %]" data-for-device="[% device.ip | html_entity %]" data-for-port="[% row.port | html_entity %]">
+            <i class="icon-edit nd_edit-icon"></i>
+          [% ELSE %]
+            <td nowrap class="nd_editable-cell">
           [% END %]
-        </td>
+            <div class="nd_editable-cell-content">
+              [% row.get_column(config.field) | html_entity %]
+            </div>
+            </td>
+        [% END %]
+
       [% END %]
 
       [% IF params.c_port %]
@@ -124,13 +137,26 @@
 
       [% FOREACH config IN settings._extra_device_port_cols %]
         [% NEXT UNLESS config.position == 'mid' AND params.${config.name} %]
-        <td>
+
-          [% TRY %]
+        [% TRY %]
+          <td>
           [% INCLUDE "plugin/${config.name}/device_port_column.tt" %]
-          [% CATCH %]
+          </td>
-          <!-- dummy content required by Template Toolkit TRY -->
+        [% CATCH %]
+          [% CLEAR %]
+          [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %]
+            <td nowrap class="nd_editable-cell" contenteditable="true"
+                data-field="[% config.field | html_entity %]" data-for-device="[% device.ip | html_entity %]" data-for-port="[% row.port | html_entity %]">
+            <i class="icon-edit nd_edit-icon"></i>
+          [% ELSE %]
+            <td nowrap class="nd_editable-cell">
           [% END %]
-        </td>
+            <div class="nd_editable-cell-content">
+              [% row.get_column(config.field) | html_entity %]
+            </div>
+            </td>
+        [% END %]
+
       [% END %]
 
       [% IF params.c_descr %]
@@ -408,13 +434,26 @@
 
       [% FOREACH config IN settings._extra_device_port_cols %]
         [% NEXT UNLESS config.position == 'right' AND params.${config.name} %]
-        <td>
+
-          [% TRY %]
+        [% TRY %]
+          <td>
           [% INCLUDE "plugin/${config.name}/device_port_column.tt" %]
-          [% CATCH %]
+          </td>
-          <!-- dummy content required by Template Toolkit TRY -->
+        [% CATCH %]
+          [% CLEAR %]
+          [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %]
+            <td nowrap class="nd_editable-cell" contenteditable="true"
+                data-field="[% config.field | html_entity %]" data-for-device="[% device.ip | html_entity %]" data-for-port="[% row.port | html_entity %]">
+            <i class="icon-edit nd_edit-icon"></i>
+          [% ELSE %]
+            <td nowrap class="nd_editable-cell">
           [% END %]
-        </td>
+            <div class="nd_editable-cell-content">
+              [% row.get_column(config.field) | html_entity %]
+            </div>
+            </td>
+        [% END %]
+
       [% END %]
     </tr>
     [% END %]

share/views/ajax/device/ports_csv.tt

@@ -40,7 +40,7 @@
     [% TRY %]
     [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %]
     [% CATCH %]
-    [% myport.push('') %]
+    [% myport.push( row.get_column(config.field) ) %]
     [% END %]
   [% END %]
 
@@ -53,7 +53,7 @@
     [% TRY %]
     [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %]
     [% CATCH %]
-    [% myport.push('') %]
+    [% myport.push( row.get_column(config.field) ) %]
     [% END %]
   [% END %]
 
@@ -178,7 +178,7 @@
     [% TRY %]
     [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %]
     [% CATCH %]
-    [% myport.push('') %]
+    [% myport.push( row.get_column(config.field) ) %]
     [% END %]
   [% END %]