From 1c7c749f0e3f14639ad0cc849f70b8365af665a1 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 9 Dec 2022 10:20:26 +0000 Subject: [PATCH] custom fields on devices and ports in the web from config (#945) * custom device field web display and edit * make display work; relies on T::T calling dict slot or method with same syntax * add storing port custom fields * use resultset method instead, use cf_ prefix * update Pg min ver for jsonb * allow override of position and default for port custom fields * support hidden for custom fields * update description of Objects API class * allow left and mid position for custom fields * add custom fields in csv * change port control sidebar label * fix default missing bug on backend jobs --- lib/App/Netdisco.pm | 2 +- lib/App/Netdisco/DB.pm | 2 +- lib/App/Netdisco/DB/Result/Device.pm | 2 + lib/App/Netdisco/DB/Result/DevicePort.pm | 2 + lib/App/Netdisco/JobQueue/PostgreSQL.pm | 50 ++++++++--- lib/App/Netdisco/Web.pm | 3 +- lib/App/Netdisco/Web/CustomFields.pm | 86 +++++++++++++++++++ lib/App/Netdisco/Web/Plugin/Device/Details.pm | 3 +- lib/App/Netdisco/Web/Plugin/Device/Ports.pm | 2 +- lib/App/Netdisco/Web/PortControl.pm | 3 +- share/config.yml | 5 +- .../App-Netdisco-DB-76-77-PostgreSQL.sql | 11 +++ share/views/ajax/device/details.tt | 23 +++-- share/views/ajax/device/ports.tt | 69 +++++++++++---- share/views/ajax/device/ports_csv.tt | 6 +- 15 files changed, 225 insertions(+), 44 deletions(-) create mode 100644 lib/App/Netdisco/Web/CustomFields.pm create mode 100644 share/schema_versions/App-Netdisco-DB-76-77-PostgreSQL.sql diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index aa383b92..4c59b02a 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -103,7 +103,7 @@ will take about 250MB including MIB files. root:~# useradd -m -p x -s /bin/bash netdisco Netdisco uses the PostgreSQL database server. Install PostgreSQL (at least -version 9.4) and then change to the PostgreSQL superuser (usually +version 9.6) and then change to the PostgreSQL superuser (usually C). Create a new database and PostgreSQL user for the Netdisco application: diff --git a/lib/App/Netdisco/DB.pm b/lib/App/Netdisco/DB.pm index b8f8f650..9e12a51d 100644 --- a/lib/App/Netdisco/DB.pm +++ b/lib/App/Netdisco/DB.pm @@ -11,7 +11,7 @@ __PACKAGE__->load_namespaces( ); our # try to hide from kwalitee - $VERSION = 76; # schema version used for upgrades, keep as integer + $VERSION = 77; # schema version used for upgrades, keep as integer use Path::Class; use File::ShareDir 'dist_dir'; diff --git a/lib/App/Netdisco/DB/Result/Device.pm b/lib/App/Netdisco/DB/Result/Device.pm index 5eeb8776..e3834896 100644 --- a/lib/App/Netdisco/DB/Result/Device.pm +++ b/lib/App/Netdisco/DB/Result/Device.pm @@ -85,6 +85,8 @@ __PACKAGE__->add_columns( { data_type => "boolean", is_nullable => 0, default_value => \"false" }, "pae_is_enabled", { data_type => "boolean", is_nullable => 1 }, + "custom_fields", + { data_type => "jsonb", is_nullable => 0, default_value => \"{}" }, ); __PACKAGE__->set_primary_key("ip"); diff --git a/lib/App/Netdisco/DB/Result/DevicePort.pm b/lib/App/Netdisco/DB/Result/DevicePort.pm index db574a79..3d8da347 100644 --- a/lib/App/Netdisco/DB/Result/DevicePort.pm +++ b/lib/App/Netdisco/DB/Result/DevicePort.pm @@ -71,6 +71,8 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 1 }, "lastchange", { data_type => "bigint", is_nullable => 1 }, + "custom_fields", + { data_type => "jsonb", is_nullable => 0, default_value => \"{}" }, ); __PACKAGE__->set_primary_key("port", "ip"); diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index 8468d805..0525d071 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -332,18 +332,44 @@ sub jq_insert { my $happy = false; try { schema(vars->{'tenant'})->txn_do(sub { - schema(vars->{'tenant'})->resultset('Admin')->populate([ - map {{ - device => $_->{device}, - device_key => $_->{device_key}, - port => $_->{port}, - action => $_->{action}, - subaction => ($_->{extra} || $_->{subaction}), - username => $_->{username}, - userip => $_->{userip}, - status => 'queued', - }} @$jobs - ]); + if (scalar @$jobs == 1 and defined $jobs->[0]->{device} and + scalar grep {$_ eq $jobs->[0]->{action}} @{ setting('_inline_actions') || [] }) { + + my $spec = $jobs->[0]; + my $row = undef; + + if ($spec->{port}) { + $row = schema(vars->{'tenant'})->resultset('DevicePort') + ->find($spec->{port}, $spec->{device}); + } + else { + $row = schema(vars->{'tenant'})->resultset('Device') + ->find($spec->{device}); + } + die 'failed to find row for custom field update' unless $row; + + $spec->{action} =~ s/^cf_//; + $spec->{subaction} = to_json( $spec->{subaction} ); + $row->make_column_dirty('custom_fields'); + $row->update({ + custom_fields => \['jsonb_set(custom_fields, ?, ?)' + => (qq{{$spec->{action}}}, $spec->{subaction}) ] + })->discard_changes(); + } + else { + schema(vars->{'tenant'})->resultset('Admin')->populate([ + map {{ + device => $_->{device}, + device_key => $_->{device_key}, + port => $_->{port}, + action => $_->{action}, + subaction => ($_->{extra} || $_->{subaction}), + username => $_->{username}, + userip => $_->{userip}, + status => 'queued', + }} @$jobs + ]); + } }); $happy = true; } diff --git a/lib/App/Netdisco/Web.pm b/lib/App/Netdisco/Web.pm index 24566126..96d56e7f 100644 --- a/lib/App/Netdisco/Web.pm +++ b/lib/App/Netdisco/Web.pm @@ -111,6 +111,7 @@ use App::Netdisco::Web::TypeAhead; use App::Netdisco::Web::PortControl; use App::Netdisco::Web::Statistics; use App::Netdisco::Web::Password; +use App::Netdisco::Web::CustomFields; use App::Netdisco::Web::GenericReport; sub _load_web_plugins { @@ -391,7 +392,7 @@ $swagger_doc->{tags} = [ {name => 'Search', description => 'Search Operations'}, {name => 'Objects', - description => 'Retrieve Device, Port, and associated Node Data'}, + description => 'Device, Port, and associated Node Data'}, {name => 'Reports', description => 'Canned and Custom Reports'}, ]; diff --git a/lib/App/Netdisco/Web/CustomFields.pm b/lib/App/Netdisco/Web/CustomFields.pm new file mode 100644 index 00000000..05dfc5b4 --- /dev/null +++ b/lib/App/Netdisco/Web/CustomFields.pm @@ -0,0 +1,86 @@ +package App::Netdisco::Web::CustomFields; + +use Dancer ':syntax'; +use Dancer::Plugin::DBIC; + +use App::Netdisco::DB::ResultSet::Device; +use App::Netdisco::DB::ResultSet::DevicePort; + +use App::Netdisco::Web::Plugin; + +my @inline_device_actions = (); +my @inline_device_port_actions = (); + +foreach my $config (@{ setting('custom_fields')->{'device'} || [] }) { + + if (! $config->{'name'}) { + error 'custom_field missing name'; + next; + } + + register_device_details({ + %{ $config }, + field => ('cf_' . $config->{'name'}), + label => ($config->{'label'} || ucfirst $config->{'name'}), + }) unless $config->{'hidden'}; + + push @inline_device_actions, $config->{'name'}; + +} + +foreach my $config (@{ setting('custom_fields')->{'device_port'} || [] }) { + + if (! $config->{'name'}) { + error 'custom_field missing name'; + next; + } + + register_device_port_column({ + position => 'right', # or "mid" or "right" + default => undef, # or undef + %{ $config }, + field => ('cf_' . $config->{'name'}), + label => ($config->{'label'} || ucfirst $config->{'name'}), + }) unless $config->{'hidden'}; + + push @inline_device_port_actions, $config->{'name'}; + +} + +{ + package App::Netdisco::DB::ResultSet::Device; + + sub with_custom_fields { + my ($rs, $cond, $attrs) = @_; + + return $rs + ->search_rs($cond, $attrs) + ->search({}, + { '+columns' => { + map {( ('cf_'. $_) => \[ 'me.custom_fields ->> ?' => $_ ] )} + @inline_device_actions + }}); + } +} + +{ + package App::Netdisco::DB::ResultSet::DevicePort; + + sub with_custom_fields { + my ($rs, $cond, $attrs) = @_; + + return $rs + ->search_rs($cond, $attrs) + ->search({}, + { '+columns' => { + map {( ('cf_'. $_) => \[ 'me.custom_fields ->> ?' => $_ ] )} + @inline_device_port_actions + }}); + } +} + +set('_inline_actions' => [ + map {'cf_' . $_} (@inline_device_actions, @inline_device_port_actions) +]); + +true; diff --git a/lib/App/Netdisco/Web/Plugin/Device/Details.pm b/lib/App/Netdisco/Web/Plugin/Device/Details.pm index 5ddbfdac..24ff4d02 100644 --- a/lib/App/Netdisco/Web/Plugin/Device/Details.pm +++ b/lib/App/Netdisco/Web/Plugin/Device/Details.pm @@ -23,8 +23,7 @@ ajax '/ajax/content/device/details' => require_login sub { '+as' => ['has_snapshot'], join => 'snapshot', }, - )->with_times() - ->hri->all; + )->with_times->with_custom_fields->hri->all; my @power = schema(vars->{'tenant'})->resultset('DevicePower') diff --git a/lib/App/Netdisco/Web/Plugin/Device/Ports.pm b/lib/App/Netdisco/Web/Plugin/Device/Ports.pm index 4a3b0179..c403c30f 100644 --- a/lib/App/Netdisco/Web/Plugin/Device/Ports.pm +++ b/lib/App/Netdisco/Web/Plugin/Device/Ports.pm @@ -20,7 +20,7 @@ get '/ajax/content/device/ports' => require_login sub { my $device = schema(vars->{'tenant'})->resultset('Device') ->search_for_device($q) or send_error('Bad device', 400); - my $set = $device->ports->with_properties; + my $set = $device->ports->with_properties->with_custom_fields; # refine by ports if requested my $f = param('f'); diff --git a/lib/App/Netdisco/Web/PortControl.pm b/lib/App/Netdisco/Web/PortControl.pm index 9c3c6686..382fb049 100644 --- a/lib/App/Netdisco/Web/PortControl.pm +++ b/lib/App/Netdisco/Web/PortControl.pm @@ -24,7 +24,7 @@ ajax '/ajax/portcontrol' => require_any_role [qw(admin port_control)] => sub { 'c_power' => 'power', ); - my $action = $action_map{ param('field') }; + my $action = ($action_map{ param('field') } || param('field') || ''); my $subaction = ($action =~ m/^(?:power|portcontrol)/ ? (param('action') ."-other") : param('value')); @@ -34,6 +34,7 @@ ajax '/ajax/portcontrol' => require_any_role [qw(admin port_control)] => sub { my $act = "$action $subaction"; $act =~ s/-other$//; $act =~ s/^portcontrol/port/; + $act =~ s/^device_port_custom_field_/custom_field: /; schema(vars->{'tenant'})->resultset('DevicePortLog')->create({ ip => param('device'), diff --git a/share/config.yml b/share/config.yml index 6a47f090..99cb2bc5 100644 --- a/share/config.yml +++ b/share/config.yml @@ -112,7 +112,7 @@ sidebar_defaults: search_device: matchall: { default: checked } device_ports: - c_admin: { label: 'Port Controls', default: null, idx: 0 } + c_admin: { label: 'Port Control and Editing', default: null, idx: 0 } c_port: { label: 'Port', default: checked, idx: 1 } c_descr: { label: 'Description', default: null, idx: 2 } c_comment: { label: 'Last Comment', default: null, idx: 3 } @@ -275,6 +275,9 @@ community_rw: [] device_auth: [] use_legacy_rancidexport: false use_legacy_sshcollector: false +custom_fields: + device: [] + device_port: [] get_credentials: "" bulkwalk_off: false bulkwalk_no: [] diff --git a/share/schema_versions/App-Netdisco-DB-76-77-PostgreSQL.sql b/share/schema_versions/App-Netdisco-DB-76-77-PostgreSQL.sql new file mode 100644 index 00000000..9f793bb1 --- /dev/null +++ b/share/schema_versions/App-Netdisco-DB-76-77-PostgreSQL.sql @@ -0,0 +1,11 @@ +BEGIN; + +ALTER TABLE device ADD COLUMN "custom_fields" jsonb DEFAULT '{}'; + +UPDATE device SET custom_fields = '{}'; + +ALTER TABLE device_port ADD COLUMN "custom_fields" jsonb DEFAULT '{}'; + +UPDATE device_port SET custom_fields = '{}'; + +COMMIT; diff --git a/share/views/ajax/device/details.tt b/share/views/ajax/device/details.tt index 3dda247f..8cfc1675 100644 --- a/share/views/ajax/device/details.tt +++ b/share/views/ajax/device/details.tt @@ -79,14 +79,25 @@ [% config.label | html_entity %] - - - [% TRY %] - [% INCLUDE "plugin/${config.name}/device_details.tt" %] - [% CATCH %] - + [% IF config.editable AND user_can_port_control %] + [% END %] + [% TRY %] + + [% INCLUDE "plugin/${config.name}/device_details.tt" %] + + [% CATCH %] + [% CLEAR %] + [% IF config.editable AND user_can_port_control %] + + [% ELSE %] + + [% END %] + [% d.${config.field} | html_entity %] + + [% END %] [% END %] [% END %] diff --git a/share/views/ajax/device/ports.tt b/share/views/ajax/device/ports.tt index a78e1ec7..80955ca6 100644 --- a/share/views/ajax/device/ports.tt +++ b/share/views/ajax/device/ports.tt @@ -60,13 +60,26 @@ [% FOREACH config IN settings._extra_device_port_cols %] [% NEXT UNLESS config.position == 'left' AND params.${config.name} %] - - [% TRY %] + + [% TRY %] + [% INCLUDE "plugin/${config.name}/device_port_column.tt" %] - [% CATCH %] - + + [% CATCH %] + [% CLEAR %] + [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %] + + + [% ELSE %] + [% END %] - +
+ [% row.get_column(config.field) | html_entity %] +
+ + [% END %] + [% END %] [% IF params.c_port %] @@ -124,13 +137,26 @@ [% FOREACH config IN settings._extra_device_port_cols %] [% NEXT UNLESS config.position == 'mid' AND params.${config.name} %] - - [% TRY %] + + [% TRY %] + [% INCLUDE "plugin/${config.name}/device_port_column.tt" %] - [% CATCH %] - + + [% CATCH %] + [% CLEAR %] + [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %] + + + [% ELSE %] + [% END %] - +
+ [% row.get_column(config.field) | html_entity %] +
+ + [% END %] + [% END %] [% IF params.c_descr %] @@ -408,13 +434,26 @@ [% FOREACH config IN settings._extra_device_port_cols %] [% NEXT UNLESS config.position == 'right' AND params.${config.name} %] - - [% TRY %] + + [% TRY %] + [% INCLUDE "plugin/${config.name}/device_port_column.tt" %] - [% CATCH %] - + + [% CATCH %] + [% CLEAR %] + [% IF config.editable AND user_can_port_control AND params.c_admin AND row.portctl %] + + + [% ELSE %] + [% END %] - +
+ [% row.get_column(config.field) | html_entity %] +
+ + [% END %] + [% END %] [% END %] diff --git a/share/views/ajax/device/ports_csv.tt b/share/views/ajax/device/ports_csv.tt index 42d756a3..7827b0fd 100644 --- a/share/views/ajax/device/ports_csv.tt +++ b/share/views/ajax/device/ports_csv.tt @@ -40,7 +40,7 @@ [% TRY %] [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %] [% CATCH %] - [% myport.push('') %] + [% myport.push( row.get_column(config.field) ) %] [% END %] [% END %] @@ -53,7 +53,7 @@ [% TRY %] [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %] [% CATCH %] - [% myport.push('') %] + [% myport.push( row.get_column(config.field) ) %] [% END %] [% END %] @@ -178,7 +178,7 @@ [% TRY %] [% PROCESS "plugin/${config.name}/device_port_column_csv.tt" %] [% CATCH %] - [% myport.push('') %] + [% myport.push( row.get_column(config.field) ) %] [% END %] [% END %]