This commit adds a table 'device_skip' that is used to restrict job queue searches to avoid jobs that are not permitted on this backend via *_no ACLs, or jobs on devices that have previously encountered multiple SNMP timeouts. When the backend loads or a device is added, a row is added to the table if that device should not be polled on this backend (together with the job actions which are to be skipped/denied). When a device SNMP connect fails a counter in the same row (or a new row) is incremented. There is also a new report 'SNMP Connect Failures' to show the devices with non-zero SNMP connect failure counters. A configurable limit in the setting 'max_deferrals' is used to set the threshold of no longer polling the device. To reset the deferrals/failures count, restart the Netdisco backend (which regenerates 'device_skip' cache entries). Squashed commit of the following: commitb5e32c219dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:55:14 2017 +0100 show all failed connections in report commitffce3cee84Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:12:39 2017 +0100 only resolve fqdn once commitcc4f680f01Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:10:20 2017 +0100 Revert "only resolve fqdn once" This reverts commit3d136a54de. commitd8d082b30eAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:09:05 2017 +0100 a report to show SNMP failures commit3d136a54deAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 19:37:58 2017 +0100 only resolve fqdn once commit4550b8a84cAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 17:27:43 2017 +0100 skipover now implicit from deferrals/actionset; fix sql where logic with better correlation commitb51edbccd2Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 16:11:29 2017 +0100 only abort lock if action matches badactions commit415559b24fAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:56:42 2017 +0100 set skipover true when adding to actionset commit1086f2c467Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:50:56 2017 +0100 fix empty actionset commit31962580b8Merge:9b2e993e6808133bAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:25:08 2017 +0100 Merge branch 'og-device_skip' of github.com:netdisco/netdisco into og-device_skip commit6808133bdbAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:19:54 2017 +0100 in-job checks for acls are required for netdisco-do foreground actions commit3944dd7813Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:18:30 2017 +0100 avoid extra device lookup commit9b2e993e0fAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 12:31:36 2017 +0100 also delete device_skip rows when deleting device commitb55854e91dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 11:34:27 2017 +0100 actions in device_skip table are now an array/set commit5e126eef07Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 09:36:33 2017 +0100 typo commit44266f2767Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 09:14:25 2017 +0100 *able checks within jobs should not be necessary with skiplist commite7c22e7d11Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:58:57 2017 +0100 increment deferrals field when job is deferred commit88ae9c00baAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:40:27 2017 +0100 turn connect fail into defer commiteac1857043Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:26:59 2017 +0100 rename failures column to be deferrals commit96ed444bbbAuthor: Oliver Gorwits <oliver@cpan.org> Date: Mon May 22 22:52:51 2017 +0100 set up list of jobs the backend instance should skip commit3a0019296dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Mon May 22 22:01:50 2017 +0100 separate out is_*able last_* checks commitcf8589aba2Author: Oliver Gorwits <oliver@cpan.org> Date: Sun May 21 22:35:38 2017 +0100 change from ignore to skip name commited193356f8Author: Oliver Gorwits <oliver@cpan.org> Date: Sun May 21 14:52:33 2017 +0100 device_ignore table to track devices to skip in polling
		
			
				
	
	
		
			199 lines
		
	
	
		
			5.5 KiB
		
	
	
	
		
			Perl
		
	
	
	
	
	
			
		
		
	
	
			199 lines
		
	
	
		
			5.5 KiB
		
	
	
	
		
			Perl
		
	
	
	
	
	
| package App::Netdisco::DB::ResultSet;
 | |
| 
 | |
| use strict;
 | |
| use warnings;
 | |
| 
 | |
| use base 'DBIx::Class::ResultSet';
 | |
| 
 | |
| __PACKAGE__->load_components(qw/
 | |
|   Helper::ResultSet::SetOperations
 | |
|   Helper::ResultSet::Shortcut
 | |
|   Helper::ResultSet::CorrelateRelationship
 | |
| /);
 | |
| 
 | |
| =head1 ADDITIONAL METHODS
 | |
| 
 | |
| =head2 get_distinct_col( $column )
 | |
| 
 | |
| Returns an asciibetical sorted list of the distinct values in the given column
 | |
| of the Device table. This is useful for web forms when you want to provide a
 | |
| drop-down list of possible options.
 | |
| 
 | |
| =cut
 | |
| 
 | |
| sub get_distinct_col {
 | |
|     my ( $rs, $col ) = @_;
 | |
|     return $rs unless $col;
 | |
| 
 | |
|     return $rs->search(
 | |
|         {},
 | |
|         {   columns  => [$col],
 | |
|             order_by => $col,
 | |
|             distinct => 1
 | |
|         }
 | |
|     )->get_column($col)->all;
 | |
| }
 | |
| 
 | |
| =head2 get_datatables_data( $params )
 | |
| 
 | |
| Returns a ResultSet for DataTables Server-side processing which populates
 | |
| the displayed table.  Evaluates the supplied query parameters for filtering,
 | |
| paging, and ordering information.  Note: query paramters are expected to be
 | |
| passed as a reference to an expanded hash of hashes.
 | |
| 
 | |
| Filtering if present, will generate simple LIKE matching conditions for each
 | |
| searchable column (searchability indicated by query parameters) after each
 | |
| column is casted to text.  Conditions are combined as disjunction (OR).
 | |
| Note: this does not match the built-in DataTables filtering which does it
 | |
| word by word on any field. 
 | |
| 
 | |
| =cut
 | |
| 
 | |
| sub get_datatables_data {
 | |
|     my $rs     = shift;
 | |
|     my $params = shift;
 | |
|     my $attrs  = shift;
 | |
| 
 | |
|     die "condition parameter to search_by_field must be hashref\n"
 | |
|         if ref {} ne ref $params
 | |
|             or 0 == scalar keys %$params;
 | |
| 
 | |
|     # -- Paging
 | |
|     $rs = $rs->_with_datatables_paging($params);
 | |
| 
 | |
|     # -- Ordering
 | |
|     $rs = $rs->_with_datatables_order_clause($params);
 | |
| 
 | |
|     # -- Filtering
 | |
|     $rs = $rs->_with_datatables_where_clause($params);
 | |
| 
 | |
|     return $rs;
 | |
| }
 | |
| 
 | |
| =head2 get_datatables_filtered_count( $params )
 | |
| 
 | |
| Returns the total records, after filtering (i.e. the total number of
 | |
| records after filtering has been applied - not just the number of records
 | |
| being returned for this page of data) for a datatables ResultSet and
 | |
| query parameters.  Note: query paramters are expected to be passed as a
 | |
| reference to an expanded hash of hashes.
 | |
| 
 | |
| =cut
 | |
| 
 | |
| sub get_datatables_filtered_count {
 | |
|     my $rs     = shift;
 | |
|     my $params = shift;
 | |
| 
 | |
|     return $rs->_with_datatables_where_clause($params)->count;
 | |
| 
 | |
| }
 | |
| 
 | |
| sub _with_datatables_order_clause {
 | |
|     my $rs     = shift;
 | |
|     my $params = shift;
 | |
|     my $attrs  = shift;
 | |
| 
 | |
|     my @order = ();
 | |
| 
 | |
|     if ( defined $params->{'order'}{0} ) {
 | |
|         for ( my $i = 0; $i < (scalar keys %{$params->{'order'}}); $i++ ) {
 | |
| 
 | |
|            # build direction, must be '-asc' or '-desc' (cf. SQL::Abstract)
 | |
|            # we only get 'asc' or 'desc', so they have to be prefixed with '-'
 | |
|             my $direction = '-' . $params->{'order'}{$i}{'dir'};
 | |
| 
 | |
|             # We only get the column index (starting from 0), so we have to
 | |
|             # translate the index into a column name.
 | |
|             my $column_name = _datatables_index_to_column( $params,
 | |
|                 $params->{'order'}{$i}{'column'} );
 | |
| 
 | |
|             # Prefix with table alias if no prefix
 | |
|             my $csa = $rs->current_source_alias;
 | |
|             $column_name =~ s/^(\w+)$/$csa\.$1/x;
 | |
|             push @order, { $direction => $column_name };
 | |
|         }
 | |
|     }
 | |
| 
 | |
|     $rs = $rs->order_by( \@order );
 | |
|     return $rs;
 | |
| }
 | |
| 
 | |
| # NOTE this does not match the built-in DataTables filtering which does it
 | |
| # word by word on any field.
 | |
| #
 | |
| # General filtering using LIKE, this will not be efficient as is will not
 | |
| # be able to use indexes.
 | |
| 
 | |
| sub _with_datatables_where_clause {
 | |
|     my $rs     = shift;
 | |
|     my $params = shift;
 | |
|     my $attrs  = shift;
 | |
| 
 | |
|     my %where = ();
 | |
| 
 | |
|     if ( defined $params->{'search'}{'value'}
 | |
|         && $params->{'search'}{'value'} )
 | |
|     {
 | |
|         my $search_string = $params->{'search'}{'value'};
 | |
|         for ( my $i = 0; $i < (scalar keys %{$params->{'columns'}}); $i++ ) {
 | |
| 
 | |
|            # Iterate over each column and check if it is searchable.
 | |
|            # If so, add a constraint to the where clause restricting the given
 | |
|            # column. In the query, the column is identified by it's index, we
 | |
|            # need to translate the index to the column name.
 | |
|             if (    $params->{'columns'}{$i}{'searchable'}
 | |
|                 and $params->{'columns'}{$i}{'searchable'} eq 'true' )
 | |
|             {
 | |
|                 my $column = _datatables_index_to_column( $params, $i );
 | |
|                 my $csa = $rs->current_source_alias;
 | |
|                 $column =~ s/^(\w+)$/$csa\.$1/x;
 | |
| 
 | |
|                 # Cast everything to text for LIKE search
 | |
|                 $column = $column . '::text';
 | |
|                 push @{ $where{'-or'} },
 | |
|                     { $column => { -like => '%' . $search_string . '%' } };
 | |
|             }
 | |
|         }
 | |
|     }
 | |
| 
 | |
|     $rs = $rs->search( \%where, $attrs );
 | |
|     return $rs;
 | |
| }
 | |
| 
 | |
| sub _with_datatables_paging {
 | |
|     my $rs     = shift;
 | |
|     my $params = shift;
 | |
|     my $attrs  = shift;
 | |
| 
 | |
|     my $limit = $params->{'length'};
 | |
| 
 | |
|     my $offset = 0;
 | |
|     if ( defined $params->{'start'} && $params->{'start'} ) {
 | |
|         $offset = $params->{'start'};
 | |
|     }
 | |
|     $attrs->{'offset'} = $offset;
 | |
| 
 | |
|     $rs = $rs->search( {}, $attrs );
 | |
|     $rs = $rs->limit($limit) if ($limit and $limit > 0);
 | |
| 
 | |
|     return $rs;
 | |
| }
 | |
| 
 | |
| # Use the DataTables columns.data definition to derive the column
 | |
| # name from the index.
 | |
| 
 | |
| sub _datatables_index_to_column {
 | |
|     my $params = shift;
 | |
|     my $i      = shift;
 | |
| 
 | |
|     my $field;
 | |
| 
 | |
|     if ( !defined($i) ) {
 | |
|         $i = 0;
 | |
|     }
 | |
|     $field = $params->{'columns'}{$i}{'data'};
 | |
|     return $field;
 | |
| }
 | |
| 
 | |
| 1;
 |