This commit adds a table 'device_skip' that is used to restrict job queue searches to avoid jobs that are not permitted on this backend via *_no ACLs, or jobs on devices that have previously encountered multiple SNMP timeouts. When the backend loads or a device is added, a row is added to the table if that device should not be polled on this backend (together with the job actions which are to be skipped/denied). When a device SNMP connect fails a counter in the same row (or a new row) is incremented. There is also a new report 'SNMP Connect Failures' to show the devices with non-zero SNMP connect failure counters. A configurable limit in the setting 'max_deferrals' is used to set the threshold of no longer polling the device. To reset the deferrals/failures count, restart the Netdisco backend (which regenerates 'device_skip' cache entries). Squashed commit of the following: commitb5e32c219dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:55:14 2017 +0100 show all failed connections in report commitffce3cee84Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:12:39 2017 +0100 only resolve fqdn once commitcc4f680f01Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:10:20 2017 +0100 Revert "only resolve fqdn once" This reverts commit3d136a54de. commitd8d082b30eAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 20:09:05 2017 +0100 a report to show SNMP failures commit3d136a54deAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 19:37:58 2017 +0100 only resolve fqdn once commit4550b8a84cAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 17:27:43 2017 +0100 skipover now implicit from deferrals/actionset; fix sql where logic with better correlation commitb51edbccd2Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 16:11:29 2017 +0100 only abort lock if action matches badactions commit415559b24fAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:56:42 2017 +0100 set skipover true when adding to actionset commit1086f2c467Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:50:56 2017 +0100 fix empty actionset commit31962580b8Merge:9b2e993e6808133bAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:25:08 2017 +0100 Merge branch 'og-device_skip' of github.com:netdisco/netdisco into og-device_skip commit6808133bdbAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:19:54 2017 +0100 in-job checks for acls are required for netdisco-do foreground actions commit3944dd7813Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 13:18:30 2017 +0100 avoid extra device lookup commit9b2e993e0fAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 12:31:36 2017 +0100 also delete device_skip rows when deleting device commitb55854e91dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 11:34:27 2017 +0100 actions in device_skip table are now an array/set commit5e126eef07Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 09:36:33 2017 +0100 typo commit44266f2767Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 09:14:25 2017 +0100 *able checks within jobs should not be necessary with skiplist commite7c22e7d11Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:58:57 2017 +0100 increment deferrals field when job is deferred commit88ae9c00baAuthor: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:40:27 2017 +0100 turn connect fail into defer commiteac1857043Author: Oliver Gorwits <oliver@cpan.org> Date: Tue May 23 08:26:59 2017 +0100 rename failures column to be deferrals commit96ed444bbbAuthor: Oliver Gorwits <oliver@cpan.org> Date: Mon May 22 22:52:51 2017 +0100 set up list of jobs the backend instance should skip commit3a0019296dAuthor: Oliver Gorwits <oliver@cpan.org> Date: Mon May 22 22:01:50 2017 +0100 separate out is_*able last_* checks commitcf8589aba2Author: Oliver Gorwits <oliver@cpan.org> Date: Sun May 21 22:35:38 2017 +0100 change from ignore to skip name commited193356f8Author: Oliver Gorwits <oliver@cpan.org> Date: Sun May 21 14:52:33 2017 +0100 device_ignore table to track devices to skip in polling
		
			
				
	
	
		
			199 lines
		
	
	
		
			5.5 KiB
		
	
	
	
		
			Perl
		
	
	
	
	
	
			
		
		
	
	
			199 lines
		
	
	
		
			5.5 KiB
		
	
	
	
		
			Perl
		
	
	
	
	
	
package App::Netdisco::DB::ResultSet;
 | 
						|
 | 
						|
use strict;
 | 
						|
use warnings;
 | 
						|
 | 
						|
use base 'DBIx::Class::ResultSet';
 | 
						|
 | 
						|
__PACKAGE__->load_components(qw/
 | 
						|
  Helper::ResultSet::SetOperations
 | 
						|
  Helper::ResultSet::Shortcut
 | 
						|
  Helper::ResultSet::CorrelateRelationship
 | 
						|
/);
 | 
						|
 | 
						|
=head1 ADDITIONAL METHODS
 | 
						|
 | 
						|
=head2 get_distinct_col( $column )
 | 
						|
 | 
						|
Returns an asciibetical sorted list of the distinct values in the given column
 | 
						|
of the Device table. This is useful for web forms when you want to provide a
 | 
						|
drop-down list of possible options.
 | 
						|
 | 
						|
=cut
 | 
						|
 | 
						|
sub get_distinct_col {
 | 
						|
    my ( $rs, $col ) = @_;
 | 
						|
    return $rs unless $col;
 | 
						|
 | 
						|
    return $rs->search(
 | 
						|
        {},
 | 
						|
        {   columns  => [$col],
 | 
						|
            order_by => $col,
 | 
						|
            distinct => 1
 | 
						|
        }
 | 
						|
    )->get_column($col)->all;
 | 
						|
}
 | 
						|
 | 
						|
=head2 get_datatables_data( $params )
 | 
						|
 | 
						|
Returns a ResultSet for DataTables Server-side processing which populates
 | 
						|
the displayed table.  Evaluates the supplied query parameters for filtering,
 | 
						|
paging, and ordering information.  Note: query paramters are expected to be
 | 
						|
passed as a reference to an expanded hash of hashes.
 | 
						|
 | 
						|
Filtering if present, will generate simple LIKE matching conditions for each
 | 
						|
searchable column (searchability indicated by query parameters) after each
 | 
						|
column is casted to text.  Conditions are combined as disjunction (OR).
 | 
						|
Note: this does not match the built-in DataTables filtering which does it
 | 
						|
word by word on any field. 
 | 
						|
 | 
						|
=cut
 | 
						|
 | 
						|
sub get_datatables_data {
 | 
						|
    my $rs     = shift;
 | 
						|
    my $params = shift;
 | 
						|
    my $attrs  = shift;
 | 
						|
 | 
						|
    die "condition parameter to search_by_field must be hashref\n"
 | 
						|
        if ref {} ne ref $params
 | 
						|
            or 0 == scalar keys %$params;
 | 
						|
 | 
						|
    # -- Paging
 | 
						|
    $rs = $rs->_with_datatables_paging($params);
 | 
						|
 | 
						|
    # -- Ordering
 | 
						|
    $rs = $rs->_with_datatables_order_clause($params);
 | 
						|
 | 
						|
    # -- Filtering
 | 
						|
    $rs = $rs->_with_datatables_where_clause($params);
 | 
						|
 | 
						|
    return $rs;
 | 
						|
}
 | 
						|
 | 
						|
=head2 get_datatables_filtered_count( $params )
 | 
						|
 | 
						|
Returns the total records, after filtering (i.e. the total number of
 | 
						|
records after filtering has been applied - not just the number of records
 | 
						|
being returned for this page of data) for a datatables ResultSet and
 | 
						|
query parameters.  Note: query paramters are expected to be passed as a
 | 
						|
reference to an expanded hash of hashes.
 | 
						|
 | 
						|
=cut
 | 
						|
 | 
						|
sub get_datatables_filtered_count {
 | 
						|
    my $rs     = shift;
 | 
						|
    my $params = shift;
 | 
						|
 | 
						|
    return $rs->_with_datatables_where_clause($params)->count;
 | 
						|
 | 
						|
}
 | 
						|
 | 
						|
sub _with_datatables_order_clause {
 | 
						|
    my $rs     = shift;
 | 
						|
    my $params = shift;
 | 
						|
    my $attrs  = shift;
 | 
						|
 | 
						|
    my @order = ();
 | 
						|
 | 
						|
    if ( defined $params->{'order'}{0} ) {
 | 
						|
        for ( my $i = 0; $i < (scalar keys %{$params->{'order'}}); $i++ ) {
 | 
						|
 | 
						|
           # build direction, must be '-asc' or '-desc' (cf. SQL::Abstract)
 | 
						|
           # we only get 'asc' or 'desc', so they have to be prefixed with '-'
 | 
						|
            my $direction = '-' . $params->{'order'}{$i}{'dir'};
 | 
						|
 | 
						|
            # We only get the column index (starting from 0), so we have to
 | 
						|
            # translate the index into a column name.
 | 
						|
            my $column_name = _datatables_index_to_column( $params,
 | 
						|
                $params->{'order'}{$i}{'column'} );
 | 
						|
 | 
						|
            # Prefix with table alias if no prefix
 | 
						|
            my $csa = $rs->current_source_alias;
 | 
						|
            $column_name =~ s/^(\w+)$/$csa\.$1/x;
 | 
						|
            push @order, { $direction => $column_name };
 | 
						|
        }
 | 
						|
    }
 | 
						|
 | 
						|
    $rs = $rs->order_by( \@order );
 | 
						|
    return $rs;
 | 
						|
}
 | 
						|
 | 
						|
# NOTE this does not match the built-in DataTables filtering which does it
 | 
						|
# word by word on any field.
 | 
						|
#
 | 
						|
# General filtering using LIKE, this will not be efficient as is will not
 | 
						|
# be able to use indexes.
 | 
						|
 | 
						|
sub _with_datatables_where_clause {
 | 
						|
    my $rs     = shift;
 | 
						|
    my $params = shift;
 | 
						|
    my $attrs  = shift;
 | 
						|
 | 
						|
    my %where = ();
 | 
						|
 | 
						|
    if ( defined $params->{'search'}{'value'}
 | 
						|
        && $params->{'search'}{'value'} )
 | 
						|
    {
 | 
						|
        my $search_string = $params->{'search'}{'value'};
 | 
						|
        for ( my $i = 0; $i < (scalar keys %{$params->{'columns'}}); $i++ ) {
 | 
						|
 | 
						|
           # Iterate over each column and check if it is searchable.
 | 
						|
           # If so, add a constraint to the where clause restricting the given
 | 
						|
           # column. In the query, the column is identified by it's index, we
 | 
						|
           # need to translate the index to the column name.
 | 
						|
            if (    $params->{'columns'}{$i}{'searchable'}
 | 
						|
                and $params->{'columns'}{$i}{'searchable'} eq 'true' )
 | 
						|
            {
 | 
						|
                my $column = _datatables_index_to_column( $params, $i );
 | 
						|
                my $csa = $rs->current_source_alias;
 | 
						|
                $column =~ s/^(\w+)$/$csa\.$1/x;
 | 
						|
 | 
						|
                # Cast everything to text for LIKE search
 | 
						|
                $column = $column . '::text';
 | 
						|
                push @{ $where{'-or'} },
 | 
						|
                    { $column => { -like => '%' . $search_string . '%' } };
 | 
						|
            }
 | 
						|
        }
 | 
						|
    }
 | 
						|
 | 
						|
    $rs = $rs->search( \%where, $attrs );
 | 
						|
    return $rs;
 | 
						|
}
 | 
						|
 | 
						|
sub _with_datatables_paging {
 | 
						|
    my $rs     = shift;
 | 
						|
    my $params = shift;
 | 
						|
    my $attrs  = shift;
 | 
						|
 | 
						|
    my $limit = $params->{'length'};
 | 
						|
 | 
						|
    my $offset = 0;
 | 
						|
    if ( defined $params->{'start'} && $params->{'start'} ) {
 | 
						|
        $offset = $params->{'start'};
 | 
						|
    }
 | 
						|
    $attrs->{'offset'} = $offset;
 | 
						|
 | 
						|
    $rs = $rs->search( {}, $attrs );
 | 
						|
    $rs = $rs->limit($limit) if ($limit and $limit > 0);
 | 
						|
 | 
						|
    return $rs;
 | 
						|
}
 | 
						|
 | 
						|
# Use the DataTables columns.data definition to derive the column
 | 
						|
# name from the index.
 | 
						|
 | 
						|
sub _datatables_index_to_column {
 | 
						|
    my $params = shift;
 | 
						|
    my $i      = shift;
 | 
						|
 | 
						|
    my $field;
 | 
						|
 | 
						|
    if ( !defined($i) ) {
 | 
						|
        $i = 0;
 | 
						|
    }
 | 
						|
    $field = $params->{'columns'}{$i}{'data'};
 | 
						|
    return $field;
 | 
						|
}
 | 
						|
 | 
						|
1;
 |