Merge branch 'master' into og-api-tokens

This commit is contained in:
Oliver Gorwits
2018-12-31 17:27:30 +00:00
21 changed files with 182 additions and 70 deletions

View File

@@ -18,7 +18,12 @@ script: |
ND_PHANTOMJS=/usr/bin/phantomjs ./Build test --test_files xt/
notifications:
irc:
on_success: change
on_success: always
on_failure: always
channels:
- 'chat.freenode.net#netdisco'
email:
on_success: always
on_failure: always
recipients:
- 'netdisco-ng-commit@lists.sourceforge.net'

View File

@@ -11,6 +11,9 @@ Module::Build->new(
'Module::Build' => '0.42',
},
build_requires => {
'ExtUtils::Config' => '0',
'ExtUtils::Helpers' => '0',
'ExtUtils::InstallPaths' => '0',
},
requires => {
'aliased' => '0',
@@ -69,7 +72,7 @@ Module::Build->new(
'Starman' => '0.4008',
'Storable' => '0',
'Sys::SigAction' => '0',
'SNMP::Info' => '3.61',
'SNMP::Info' => '3.64',
'SQL::Abstract' => '1.85',
'SQL::Translator' => '0.11024',
'Template' => '2.24',

37
Changes
View File

@@ -1,3 +1,40 @@
2.040002 - 2018-12-30
[BUG FIXES]
* correction for git sync
2.040001 - 2018-12-30
[ENHANCEMENTS]
* #471 OSPF neighbor discovey will try Router ID as well as Peer ID
* update dependency to SNMP::Info 3.64
[BUG FIXES]
* #422 fix for Network Map positions not saving (onlinehupe)
* fix portsort tests to work with PhantomJS 2.x
* some documentation fixes (inphobia)
2.040000 - 2018-12-28
[NEW FEATURES]
* #464 ignore_notpresent_types config to ignore notPresent interfaces (inphobia, nic)
* #469 jobs_qdepth config to set length of job queue in web view (inphobia)
[ENHANCEMENTS]
* #459 additional interface type to ignore (inphobia)
* #460 several improvements to netdisco-sshcollector (rc9000)
[BUG FIXES]
* #456 fix POD error (manwar)
* #461 provide stdin to sshcollector (rc9000)
* #467 documentation fixes (inphobia)
2.039033 - 2018-10-19
[BUG FIXES]

View File

@@ -14,6 +14,13 @@
},
"name" : "App-Netdisco",
"prereqs" : {
"build" : {
"requires" : {
"ExtUtils::Config" : "0",
"ExtUtils::Helpers" : "0",
"ExtUtils::InstallPaths" : "0"
}
},
"configure" : {
"requires" : {
"DBIx::Class" : "0.082810",
@@ -74,7 +81,7 @@
"Plack::Middleware::ReverseProxy" : "0.15",
"Pod::Usage" : "0",
"Role::Tiny" : "1.002005",
"SNMP::Info" : "3.61",
"SNMP::Info" : "3.64",
"SQL::Abstract" : "1.85",
"SQL::Translator" : "0.11024",
"Scope::Guard" : "0",
@@ -110,7 +117,7 @@
"provides" : {
"App::Netdisco" : {
"file" : "lib/App/Netdisco.pm",
"version" : "2.039033"
"version" : "2.040002"
},
"App::Netdisco::AnyEvent::Nbtstat" : {
"file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm"
@@ -789,6 +796,6 @@
"x_IRC" : "irc://irc.freenode.org/#netdisco",
"x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users"
},
"version" : "2.039033",
"version" : "2.040002",
"x_serialization_backend" : "JSON::PP version 2.97001"
}

View File

@@ -4,6 +4,9 @@ author:
- 'Oliver Gorwits <oliver@cpan.org>'
build_requires:
Env::Path: '0'
ExtUtils::Config: '0'
ExtUtils::Helpers: '0'
ExtUtils::InstallPaths: '0'
Test::File::ShareDir::Dist: '0'
Test::More: '1.302083'
configure_requires:
@@ -19,7 +22,7 @@ name: App-Netdisco
provides:
App::Netdisco:
file: lib/App/Netdisco.pm
version: '2.039033'
version: '2.040002'
App::Netdisco::AnyEvent::Nbtstat:
file: lib/App/Netdisco/AnyEvent/Nbtstat.pm
App::Netdisco::Backend::Job:
@@ -513,7 +516,7 @@ requires:
Plack::Middleware::ReverseProxy: '0.15'
Pod::Usage: '0'
Role::Tiny: '1.002005'
SNMP::Info: '3.61'
SNMP::Info: '3.64'
SQL::Abstract: '1.85'
SQL::Translator: '0.11024'
Scope::Guard: '0'
@@ -543,5 +546,5 @@ resources:
homepage: http://netdisco.org/
license: http://opensource.org/licenses/bsd-license.php
repository: https://github.com/netdisco/netdisco
version: '2.039033'
version: '2.040002'
x_serialization_backend: 'CPAN::Meta::YAML version 0.018'

View File

@@ -50,7 +50,7 @@ netdisco-db-deploy - Database deployment for Netdisco
This script upgrades or initialises a Netdisco database schema.
~netdisco/bin/netdisco-db-deploy [--redeploy-all]
~/bin/netdisco-db-deploy [--redeploy-all]
This script connects to the database and runs without user interaction. If
there's no Nedisco schema, it is deployed. If there's an unversioned schema

View File

@@ -210,7 +210,7 @@ will cause C<netdisco-do> to run the action for all addresses in that range.
Run a discover on the device (specified with C<-d>).
~netdisco/bin/netdisco-do discover -d 192.0.2.1
~/bin/netdisco-do discover -d 192.0.2.1
=head2 discoverall
@@ -220,7 +220,7 @@ Queue a discover for all known devices.
Run a macsuck on the device (specified with C<-d>).
~netdisco/bin/netdisco-do macsuck -d 192.0.2.1
~/bin/netdisco-do macsuck -d 192.0.2.1
=head2 macwalk
@@ -230,7 +230,7 @@ Queue a macsuck for all known devices.
Run an arpnip on the device (specified with C<-d>).
~netdisco/bin/netdisco-do arpnip -d 192.0.2.1
~/bin/netdisco-do arpnip -d 192.0.2.1
=head2 arpwalk
@@ -243,9 +243,9 @@ the C<-e> parameter. Optionally request for associated nodes to be archived
(rather than deleted) by setting the C<-p> parameter to "C<yes>" (mnemonic:
B<p>reserve).
~netdisco/bin/netdisco-do delete -d 192.0.2.1
~netdisco/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun'
~netdisco/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' -p yes
~/bin/netdisco-do delete -d 192.0.2.1
~/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun'
~/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' -p yes
=head2 renumber
@@ -256,13 +256,13 @@ log and node information will also be updated to refer to the new device.
Note that I<no> check is made as to whether the new IP is reachable for future
polling.
~netdisco/bin/netdisco-do renumber -d 192.0.2.1 -e 192.0.2.254
~/bin/netdisco-do renumber -d 192.0.2.1 -e 192.0.2.254
=head2 nbtstat
Run an nbtstat on the node (specified with C<-d>).
~netdisco/bin/netdisco-do nbtstat -d 192.0.2.2
~/bin/netdisco-do nbtstat -d 192.0.2.2
=head2 nbtwalk
@@ -278,8 +278,8 @@ Archive nodes on the specified device. If you want to delete nodes, set the
C<-e> parameter to "C<no>" (mnemonic: B<e>xpire). If you want to perform the
action on a specific port, set the C<-p> parameter.
~netdisco/bin/netdisco-do expirenodes -d 192.0.2.1
~netdisco/bin/netdisco-do expirenodes -d 192.0.2.1 -p FastEthernet0/1 -e no
~/bin/netdisco-do expirenodes -d 192.0.2.1
~/bin/netdisco-do expirenodes -d 192.0.2.1 -p FastEthernet0/1 -e no
=head2 graph
@@ -289,9 +289,9 @@ You'll need to install the L<Graph::Undirected> and L<GraphViz> Perl modules,
and possibly also the C<graphviz> utility for your operating system. Also
create a directory for the output files.
mkdir ~netdisco/graph
~netdisco/bin/localenv cpanm Graph::Undirected
~netdisco/bin/localenv cpanm GraphViz
mkdir ~/graph
~/bin/localenv cpanm Graph::Undirected
~/bin/localenv cpanm GraphViz
=head2 show
@@ -303,21 +303,21 @@ If you wish to test with a device class other than that discovered, prefix the
leaf with the class short name, for example "C<Layer3::C3550::interfaces>" or
"C<Layer2::HP::uptime>".
~netdisco/bin/netdisco-do show -d 192.0.2.1 -e interfaces
~netdisco/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces
~/bin/netdisco-do show -d 192.0.2.1 -e interfaces
~/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces
A paramter may be passed to the C<SNMP::Info> method in the C<-p> parameter:
A parameter may be passed to the C<SNMP::Info> method in the C<-p> parameter:
~netdisco/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3
~/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3
=head2 psql
Start an interactive terminal with the Netdisco PostgreSQL database. If you
pass an SQL statement in the C<-e> option then it will be executed.
~netdisco/bin/netdisco-do psql
~netdisco/bin/netdisco-do psql -e 'SELECT ip, dns FROM device'
~netdisco/bin/netdisco-do psql -e 'COPY (SELECT ip, dns FROM device) TO STDOUT WITH CSV HEADER'
~/bin/netdisco-do psql
~/bin/netdisco-do psql -e 'SELECT ip, dns FROM device'
~/bin/netdisco-do psql -e 'COPY (SELECT ip, dns FROM device) TO STDOUT WITH CSV HEADER'
=head2 stats
@@ -328,44 +328,44 @@ Updates Netdisco's statistics on number of devices, nodes, etc, for today.
Set the SNMP location field on the device (specified with C<-d>). Pass the
location string in the C<-e> extra parameter.
~netdisco/bin/netdisco-do location -d 192.0.2.1 -e 'wiring closet'
~/bin/netdisco-do location -d 192.0.2.1 -e 'wiring closet'
=head2 contact
Set the SNMP contact field on the device (specified with C<-d>). Pass the
contact name in the C<-e> extra parameter.
~netdisco/bin/netdisco-do contact -d 192.0.2.1 -e 'tel: 555-2453'
~/bin/netdisco-do contact -d 192.0.2.1 -e 'tel: 555-2453'
=head2 portname
Set the description on a device port. Requires the C<-d> parameter (device),
C<-p> parameter (port), and C<-e> parameter (description).
~netdisco/bin/netdisco-do portname -d 192.0.2.1 -p FastEthernet0/1 -e 'Web Server'
~/bin/netdisco-do portname -d 192.0.2.1 -p FastEthernet0/1 -e 'Web Server'
=head2 portcontrol
Set the up/down status on a device port. Requires the C<-d> parameter
(device), C<-p> parameter (port), and C<-e> parameter ("up" or "down").
~netdisco/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e up
~netdisco/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e down
~/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e up
~/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e down
=head2 vlan
Set the native VLAN on a device port. Requires the C<-d> parameter (device),
C<-p> parameter (port), and C<-e> parameter (VLAN number).
~netdisco/bin/netdisco-do vlan -d 192.0.2.1 -p FastEthernet0/1 -e 102
~/bin/netdisco-do vlan -d 192.0.2.1 -p FastEthernet0/1 -e 102
=head2 power
Set the PoE on/off status on a device port. Requires the C<-d> parameter
(device), C<-p> parameter (port), and C<-e> parameter ("on" or "off").
~netdisco/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e on
~netdisco/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e off
~/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e on
~/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e off
=head2 dumpconfig

View File

@@ -53,10 +53,13 @@ use Pod::Usage 'pod2usage';
use Getopt::Long;
Getopt::Long::Configure ("bundling");
my ($debug, $sqltrace) = (undef, 0);
my ($debug, $sqltrace, $device, $opensshdebug, $workers) = (undef, 0, undef, undef, "auto");
my $result = GetOptions(
'debug|D' => \$debug,
'sqltrace|Q+' => \$sqltrace,
'debug|D' => \$debug,
'sqltrace|Q' => \$sqltrace,
'device|d=s' => \$device,
'opensshdebug|O' => \$opensshdebug,
'workers|w=i' => \$workers,
) or pod2usage(
-msg => 'error: bad options',
-verbose => 0,
@@ -71,10 +74,11 @@ $ENV{DBIC_TRACE} ||= $sqltrace;
# reconfigure logging to force console output
Dancer::Logger->init('console', $CONFIG);
#this may be helpful with SSH issues:
#$Net::OpenSSH::debug = ~0;
if ($opensshdebug){
$Net::OpenSSH::debug = ~0;
}
MCE::Loop::init { chunk_size => 1 };
MCE::Loop::init { chunk_size => 1, max_workers => $workers };
my %stats;
$stats{entry} = 0;
@@ -83,6 +87,11 @@ exit main();
sub main {
my @input = @{ setting('sshcollector') };
if ($device){
@input = grep{ ($_->{hostname} && $_->{hostname} eq $device)
|| ($_->{ip} && $_->{ip} eq $device) } @input;
}
#one-line Fisher-Yates from https://www.perlmonks.org/index.pl?node=Array%20One-Liners
my ($i,$j) = (0);
@input[-$i,$j] = @input[$j,-$i] while $j = rand(@input - $i), ++$i < @input;
@@ -108,7 +117,12 @@ sub main {
],
);
MCE->gather( process($hostlabel, $ssh, $host) );
if ($ssh->error){
warning "WARNING: Couldn't connect to <$hostlabel> - " . $ssh->error;
}else{
MCE->gather( process($hostlabel, $ssh, $host) );
}
}
} \@input;
@@ -170,10 +184,13 @@ full SNMP support
=head1 SYNOPSIS
# install dependencies:
~netdisco/bin/localenv cpanm --notest Net::OpenSSH Expect
~/bin/localenv cpanm --notest Net::OpenSSH Expect
# run manually, or add to cron:
~/bin/netdisco-sshcollector [-DQ]
~/bin/netdisco-sshcollector [-DQO] [-w <max_workers>]
# limit run to a single device defined in the config
~/bin/netdisco-sshcollector [-DQO] [-w <max_workers>] -d <device>
=head1 DESCRIPTION
@@ -271,10 +288,7 @@ don't support command execution via ssh:
The returned IP and MAC addresses should be in a format that the respective
B<inetaddr> and B<macaddr> datatypes in PostgreSQL can handle.
=head1 DEBUG LEVELS
The flags "C<-DQ>" can be specified, multiple times, and enable the following
items in order:
=head1 COMMAND LINE OPTIONS
=over 4
@@ -284,7 +298,21 @@ Netdisco debug log level
=item C<-Q>
L<DBIx::Class> trace enabled
L<DBIx::Class> trace enabled.
=item C<-O>
L<Net::OpenSSH> trace enabled
=item C<-w>
Set maximum parallel workers for L<MCE::Loop>. The default is B<auto>.
=item C<-d device>
Only run for a single device. Takes an IP or hostname, must exactly match the value
in the config file.
=back

View File

@@ -4,7 +4,7 @@ use strict;
use warnings;
use 5.010_000;
our $VERSION = '2.039033';
our $VERSION = '2.040002';
use App::Netdisco::Configuration;
=head1 NAME
@@ -127,7 +127,7 @@ documentation for further details.
To avoid muddying your system, use the following script to download and
install Netdisco and its dependencies into the C<netdisco> user's home area
(C<~netdisco/perl5>):
(C<~/perl5>):
su - netdisco
curl -L https://cpanmin.us/ | perl - --notest --local-lib ~/perl5 App::Netdisco
@@ -194,7 +194,7 @@ port control, etc):
~/bin/netdisco-backend start
I<note:> Whenever you upgrade your operating system, you should delete the
C<~netdisco/perl5> directory and re-run the C<curl> command above, to update
C<~/perl5> directory and re-run the C<curl> command above, to update
Netdisco's C library bindings.
I<also note:> You should take care not to run C<< netdisco-backend >> and the

View File

@@ -62,7 +62,7 @@ __PACKAGE__->set_primary_key("job");
=head2 device_skips( $backend?, $max_deferrals?, $retry_after? )
Retuns the set of C<device_skip> entries which apply to this job. They match
Returns the set of C<device_skip> entries which apply to this job. They match
the device IP, current backend, and job action.
You probably want to use the ResultSet method C<skipped> which completes this
@@ -122,7 +122,7 @@ sub display_name {
=head1 ADDITIONAL COLUMNS
=head2 entererd_stamp
=head2 entered_stamp
Formatted version of the C<entered> field, accurate to the minute.

View File

@@ -205,6 +205,8 @@ NodeIp table of nodes with addresses of the supplied IP version.
The C<version> parameter must be an integer either 4 or 6.
=back
=cut
sub ip_version {

View File

@@ -59,8 +59,9 @@ Performs initialisation of the Job Queue backend.
=head2 jq_log()
Returns a list of the most recent 50 jobs in the queue. Jobs are returned as
objects which implement the Netdisco job instance interface (see below).
Returns a list of the most recent jobs as defined in C<jobs_qdepth> from the
queue. Jobs are returned as objects which implement the Netdisco job instance
interface (see below).
=head2 jq_userlog( $user )

View File

@@ -273,7 +273,7 @@ sub jq_log {
return schema('netdisco')->resultset('Admin')->search({}, {
prefetch => 'target',
order_by => { -desc => [qw/entered device action/] },
rows => 50,
rows => (setting('jobs_qdepth') || 50),
})->with_times->hri->all;
}

View File

@@ -36,6 +36,11 @@ Returns a list of hashrefs in the format C<{ mac => MACADDR, ip => IPADDR }>.
sub arpnip {
my ($self, $hostlabel, $ssh, $args) = @_;
# IOSXR show commands seem to depend on an available STDIN
unless (-t STDIN){
open STDIN, "<", "/dev/zero" or warn "Failed to fake stdin: $!";
}
debug "$hostlabel $$ arpnip()";
my @data = $ssh->capture("show arp vrf all");

View File

@@ -16,17 +16,18 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub {
my $snmp = App::Netdisco::Transport::SNMP->reader_for($device)
or return Status->defer("discover failed: could not SNMP connect to $device");
my $ospf_peers = $snmp->ospf_peers || {};
my $bgp_peers = $snmp->bgp_peer_addr || {};
my $eigrp_peers = $snmp->eigrp_peers || {};
my $ospf_peers = $snmp->ospf_peers || {};
my $ospf_routers = $snmp->ospf_peer_id || {};
my $bgp_peers = $snmp->bgp_peer_addr || {};
my $eigrp_peers = $snmp->eigrp_peers || {};
return Status->info(" [$device] neigh - no BGP, OSPF, or EIGRP peers")
unless ((scalar values %$ospf_peers) or (scalar values %$bgp_peers)
or (scalar values %$eigrp_peers));
my $count = 0;
foreach my $ip ((values %$ospf_peers), (values %$bgp_peers),
(values %$eigrp_peers)) {
foreach my $ip ((values %$ospf_peers), (values %$ospf_routers),
(values %$bgp_peers), (values %$eigrp_peers)) {
my $peer = get_device($ip);
next if $peer->in_storage or not is_discoverable($peer);
next if vars->{'queued'}->{$ip};

View File

@@ -181,6 +181,13 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub {
next;
}
# Skip interfaces which are 'notPresent' and match the notpresent type filter
if (defined $i_up->{$entry} and defined $i_type->{$entry} and $i_up->{$entry} eq 'notPresent' and (scalar grep {$i_type->{$entry} =~ m/^$_$/} @{setting('ignore_notpresent_types') || []}) ) {
debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s) (config:ignore_notpresent_types)',
$device->ip, $entry, $port, $i_up->{$entry};
next;
}
my $lc = $i_lastchange->{$entry} || 0;
if (not $dev_uptime_wrapped and $lc > $dev_uptime) {
info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting',

View File

@@ -68,6 +68,8 @@ register_worker({ phase => 'main' }, sub {
true;
=encoding utf8
=head1 NAME
MakeRancidConf - Generate RANCID Configuration

View File

@@ -3,7 +3,9 @@
# DO NOT EDIT THIS FILE
#
# Overrides should go to ~/environments/deployment.yml
# See App::Netdisco::Manual::Configuration for explanations
#
# https://github.com/netdisco/netdisco/wiki/Configuration has
# in depth explanations about each setting.
# ----------------
# GENERAL SETTINGS
@@ -285,6 +287,7 @@ ignore_interfaces:
- 'BRI\S+-Bearer Channel'
- 'BRI\S+-Physical'
- 'BRI\S+-Signalling'
- 'BRI\S+-Signaling'
- 'Embedded-Service-Engine\d+\/\d+'
- 'Virtual-Template\d+'
- 'Virtual-Access\d+'
@@ -297,6 +300,10 @@ ignore_interfaces:
- 'Ethernet(?:-| )QOS Packet Schedu?ler'
- 'Ethernet(?:-| )WFP (?:802\.3|Native) MAC Layer Lightweight Filter'
- 'ii\d\/\d\/\d+'
ignore_notpresent_types:
- 'ethernetCsmacd'
- 'tunnel'
- 'ieee8023adLag'
ignore_private_nets: false
reverse_sysname: false
phone_capabilities:
@@ -325,6 +332,7 @@ workers:
# 50 minutes
jobs_stale_after: 3000
jobs_qdepth: 50
dns:
max_outstanding: 50

View File

@@ -2,7 +2,9 @@
# NETDISCO 2 CONFIGURATION FILE
#
# Settings in this file override share/config.yml
# See App::Netdisco::Manual::Configuration for more info.
#
# https://github.com/netdisco/netdisco/wiki/Configuration has
# in depth explanations about each setting.
# ------------------
# ESSENTIAL SETTINGS

View File

@@ -216,7 +216,7 @@ function saveMapPositions() {
graph.inspect().main.nodes.each(function(n) { n.fixed = true });
$.post(
'[% uri_for('/ajax/data/device/netmappositions') %]'
,$("#nd_vlan-entry, #nd_hgroup-select, #nd_lgroup-select, input[name='q'], input[name='mapshow']").serialize()
,$("#nd_vlan-entry, #nd_hgroup-select, #nd_lgroup-select, #nq, input[name='mapshow']").serialize()
+ '&positions=' + JSON.stringify(graph.positions())
);
toastr.success('Saved map positions.');

View File

@@ -1,3 +1,4 @@
var system = require('system');
/**
* Wait until the test condition is true or a timeout occurs. Useful for waiting
* on a server response or for a ui change (fadeIn, etc.) to occur.
@@ -34,7 +35,7 @@ function waitFor(testFx, onReady, timeOutMillis) {
};
if (phantom.args.length === 0 || phantom.args.length > 2) {
if (system.args.length === 0 || system.args.length > 2) {
console.log('Usage: run-qunit.js URL');
phantom.exit(1);
}
@@ -46,7 +47,7 @@ page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.open(phantom.args[0], function(status){
page.open(system.args[1], function(status){
if (status !== "success") {
console.log("Unable to access network");
phantom.exit(1);