From 369590d17f786a702987de7bc603a0d1c81f185c Mon Sep 17 00:00:00 2001 From: Mohammad S Anwar Date: Tue, 27 Nov 2018 11:18:45 +0000 Subject: [PATCH 01/24] Fixed pod error as reported by CPANTS. (#456) --- lib/App/Netdisco/DB/ResultSet/NodeIp.pm | 2 ++ lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/App/Netdisco/DB/ResultSet/NodeIp.pm b/lib/App/Netdisco/DB/ResultSet/NodeIp.pm index 22202dbc..e639d57b 100644 --- a/lib/App/Netdisco/DB/ResultSet/NodeIp.pm +++ b/lib/App/Netdisco/DB/ResultSet/NodeIp.pm @@ -205,6 +205,8 @@ NodeIp table of nodes with addresses of the supplied IP version. The C parameter must be an integer either 4 or 6. +=back + =cut sub ip_version { diff --git a/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm b/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm index af1a2805..8f241bae 100644 --- a/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm +++ b/lib/App/Netdisco/Worker/Plugin/MakeRancidConf.pm @@ -68,6 +68,8 @@ register_worker({ phase => 'main' }, sub { true; +=encoding utf8 + =head1 NAME MakeRancidConf - Generate RANCID Configuration From 2e0965033aec18a5d07ae8f61beeb53c82d604d4 Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Tue, 27 Nov 2018 12:19:22 +0100 Subject: [PATCH 02/24] remove reference to netdisco user in command examples. (#451) --- bin/netdisco-db-deploy | 2 +- bin/netdisco-do | 54 +++++++++++++++++++-------------------- bin/netdisco-sshcollector | 2 +- lib/App/Netdisco.pm | 4 +-- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/bin/netdisco-db-deploy b/bin/netdisco-db-deploy index a23a1d4f..51d7e8ac 100755 --- a/bin/netdisco-db-deploy +++ b/bin/netdisco-db-deploy @@ -50,7 +50,7 @@ netdisco-db-deploy - Database deployment for Netdisco This script upgrades or initialises a Netdisco database schema. - ~netdisco/bin/netdisco-db-deploy [--redeploy-all] + ~/bin/netdisco-db-deploy [--redeploy-all] This script connects to the database and runs without user interaction. If there's no Nedisco schema, it is deployed. If there's an unversioned schema diff --git a/bin/netdisco-do b/bin/netdisco-do index 9faf8c12..61c4ec07 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -210,7 +210,7 @@ will cause C to run the action for all addresses in that range. Run a discover on the device (specified with C<-d>). - ~netdisco/bin/netdisco-do discover -d 192.0.2.1 + ~/bin/netdisco-do discover -d 192.0.2.1 =head2 discoverall @@ -220,7 +220,7 @@ Queue a discover for all known devices. Run a macsuck on the device (specified with C<-d>). - ~netdisco/bin/netdisco-do macsuck -d 192.0.2.1 + ~/bin/netdisco-do macsuck -d 192.0.2.1 =head2 macwalk @@ -230,7 +230,7 @@ Queue a macsuck for all known devices. Run an arpnip on the device (specified with C<-d>). - ~netdisco/bin/netdisco-do arpnip -d 192.0.2.1 + ~/bin/netdisco-do arpnip -d 192.0.2.1 =head2 arpwalk @@ -243,9 +243,9 @@ the C<-e> parameter. Optionally request for associated nodes to be archived (rather than deleted) by setting the C<-p> parameter to "C" (mnemonic: B

reserve). - ~netdisco/bin/netdisco-do delete -d 192.0.2.1 - ~netdisco/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' - ~netdisco/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' -p yes + ~/bin/netdisco-do delete -d 192.0.2.1 + ~/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' + ~/bin/netdisco-do delete -d 192.0.2.1 -e 'older than the sun' -p yes =head2 renumber @@ -256,13 +256,13 @@ log and node information will also be updated to refer to the new device. Note that I check is made as to whether the new IP is reachable for future polling. - ~netdisco/bin/netdisco-do renumber -d 192.0.2.1 -e 192.0.2.254 + ~/bin/netdisco-do renumber -d 192.0.2.1 -e 192.0.2.254 =head2 nbtstat Run an nbtstat on the node (specified with C<-d>). - ~netdisco/bin/netdisco-do nbtstat -d 192.0.2.2 + ~/bin/netdisco-do nbtstat -d 192.0.2.2 =head2 nbtwalk @@ -278,8 +278,8 @@ Archive nodes on the specified device. If you want to delete nodes, set the C<-e> parameter to "C" (mnemonic: Bxpire). If you want to perform the action on a specific port, set the C<-p> parameter. - ~netdisco/bin/netdisco-do expirenodes -d 192.0.2.1 - ~netdisco/bin/netdisco-do expirenodes -d 192.0.2.1 -p FastEthernet0/1 -e no + ~/bin/netdisco-do expirenodes -d 192.0.2.1 + ~/bin/netdisco-do expirenodes -d 192.0.2.1 -p FastEthernet0/1 -e no =head2 graph @@ -289,9 +289,9 @@ You'll need to install the L and L Perl modules, and possibly also the C utility for your operating system. Also create a directory for the output files. - mkdir ~netdisco/graph - ~netdisco/bin/localenv cpanm Graph::Undirected - ~netdisco/bin/localenv cpanm GraphViz + mkdir ~/graph + ~/bin/localenv cpanm Graph::Undirected + ~/bin/localenv cpanm GraphViz =head2 show @@ -303,21 +303,21 @@ If you wish to test with a device class other than that discovered, prefix the leaf with the class short name, for example "C" or "C". - ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e interfaces - ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces + ~/bin/netdisco-do show -d 192.0.2.1 -e interfaces + ~/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces A paramter may be passed to the C method in the C<-p> parameter: - ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3 + ~/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3 =head2 psql Start an interactive terminal with the Netdisco PostgreSQL database. If you pass an SQL statement in the C<-e> option then it will be executed. - ~netdisco/bin/netdisco-do psql - ~netdisco/bin/netdisco-do psql -e 'SELECT ip, dns FROM device' - ~netdisco/bin/netdisco-do psql -e 'COPY (SELECT ip, dns FROM device) TO STDOUT WITH CSV HEADER' + ~/bin/netdisco-do psql + ~/bin/netdisco-do psql -e 'SELECT ip, dns FROM device' + ~/bin/netdisco-do psql -e 'COPY (SELECT ip, dns FROM device) TO STDOUT WITH CSV HEADER' =head2 stats @@ -328,44 +328,44 @@ Updates Netdisco's statistics on number of devices, nodes, etc, for today. Set the SNMP location field on the device (specified with C<-d>). Pass the location string in the C<-e> extra parameter. - ~netdisco/bin/netdisco-do location -d 192.0.2.1 -e 'wiring closet' + ~/bin/netdisco-do location -d 192.0.2.1 -e 'wiring closet' =head2 contact Set the SNMP contact field on the device (specified with C<-d>). Pass the contact name in the C<-e> extra parameter. - ~netdisco/bin/netdisco-do contact -d 192.0.2.1 -e 'tel: 555-2453' + ~/bin/netdisco-do contact -d 192.0.2.1 -e 'tel: 555-2453' =head2 portname Set the description on a device port. Requires the C<-d> parameter (device), C<-p> parameter (port), and C<-e> parameter (description). - ~netdisco/bin/netdisco-do portname -d 192.0.2.1 -p FastEthernet0/1 -e 'Web Server' + ~/bin/netdisco-do portname -d 192.0.2.1 -p FastEthernet0/1 -e 'Web Server' =head2 portcontrol Set the up/down status on a device port. Requires the C<-d> parameter (device), C<-p> parameter (port), and C<-e> parameter ("up" or "down"). - ~netdisco/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e up - ~netdisco/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e down + ~/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e up + ~/bin/netdisco-do portcontrol -d 192.0.2.1 -p FastEthernet0/1 -e down =head2 vlan Set the native VLAN on a device port. Requires the C<-d> parameter (device), C<-p> parameter (port), and C<-e> parameter (VLAN number). - ~netdisco/bin/netdisco-do vlan -d 192.0.2.1 -p FastEthernet0/1 -e 102 + ~/bin/netdisco-do vlan -d 192.0.2.1 -p FastEthernet0/1 -e 102 =head2 power Set the PoE on/off status on a device port. Requires the C<-d> parameter (device), C<-p> parameter (port), and C<-e> parameter ("on" or "off"). - ~netdisco/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e on - ~netdisco/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e off + ~/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e on + ~/bin/netdisco-do power -d 192.0.2.1 -p FastEthernet0/1 -e off =head2 dumpconfig diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 82176dc8..48d97306 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -170,7 +170,7 @@ full SNMP support =head1 SYNOPSIS # install dependencies: - ~netdisco/bin/localenv cpanm --notest Net::OpenSSH Expect + ~/bin/localenv cpanm --notest Net::OpenSSH Expect # run manually, or add to cron: ~/bin/netdisco-sshcollector [-DQ] diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 8fd841d5..2b52e45a 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -127,7 +127,7 @@ documentation for further details. To avoid muddying your system, use the following script to download and install Netdisco and its dependencies into the C user's home area -(C<~netdisco/perl5>): +(C<~/perl5>): su - netdisco curl -L https://cpanmin.us/ | perl - --notest --local-lib ~/perl5 App::Netdisco @@ -194,7 +194,7 @@ port control, etc): ~/bin/netdisco-backend start I Whenever you upgrade your operating system, you should delete the -C<~netdisco/perl5> directory and re-run the C command above, to update +C<~/perl5> directory and re-run the C command above, to update Netdisco's C library bindings. I You should take care not to run C<< netdisco-backend >> and the From 3474d028ec37c0ef813d12f806dbbd12c898909f Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Wed, 28 Nov 2018 02:43:15 +0100 Subject: [PATCH 03/24] 1 more isdn interface to ignore not sure if this is a typo or not, but bri singaling interfaces have 1 'l' on cisco devices. "BRI0-Signaling" was taken from a netdisco report, router is a "897VaK9" running ios "15.5(3)M7". as such, i've added the 1 'l' variant to the ignore_interfaces list. if this was indeed a typo perhaps the double 'll' variant can be deleted. --- share/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/share/config.yml b/share/config.yml index ae16a8dd..c1d26ea4 100644 --- a/share/config.yml +++ b/share/config.yml @@ -281,6 +281,7 @@ ignore_interfaces: - 'BRI\S+-Bearer Channel' - 'BRI\S+-Physical' - 'BRI\S+-Signalling' + - 'BRI\S+-Signaling' - 'Embedded-Service-Engine\d+\/\d+' - 'Virtual-Template\d+' - 'Virtual-Access\d+' From e94f15dc8a42baad4eb9c6771f2354dd898f7763 Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Thu, 13 Dec 2018 22:42:52 +0100 Subject: [PATCH 04/24] typo (paramter instead of parameter) typo (paramter instead of parameter) --- bin/netdisco-do | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/netdisco-do b/bin/netdisco-do index 61c4ec07..a720361b 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -306,7 +306,7 @@ leaf with the class short name, for example "C" or ~/bin/netdisco-do show -d 192.0.2.1 -e interfaces ~/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces -A paramter may be passed to the C method in the C<-p> parameter: +A parameter may be passed to the C method in the C<-p> parameter: ~/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3 From 733f82c3dc3c15115e8ba3c2a1bd2da2a3e551f5 Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Thu, 13 Dec 2018 22:52:48 +0100 Subject: [PATCH 05/24] 2 typos --- lib/App/Netdisco/DB/Result/Admin.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/DB/Result/Admin.pm b/lib/App/Netdisco/DB/Result/Admin.pm index 324f7e98..c1aebc15 100644 --- a/lib/App/Netdisco/DB/Result/Admin.pm +++ b/lib/App/Netdisco/DB/Result/Admin.pm @@ -62,7 +62,7 @@ __PACKAGE__->set_primary_key("job"); =head2 device_skips( $backend?, $max_deferrals?, $retry_after? ) -Retuns the set of C entries which apply to this job. They match +Returns the set of C entries which apply to this job. They match the device IP, current backend, and job action. You probably want to use the ResultSet method C which completes this @@ -122,7 +122,7 @@ sub display_name { =head1 ADDITIONAL COLUMNS -=head2 entererd_stamp +=head2 entered_stamp Formatted version of the C field, accurate to the minute. From 9f8dd0039bf841cc365b15956f0cd5f1d6a711ee Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Fri, 28 Dec 2018 15:32:05 +0100 Subject: [PATCH 06/24] docs are maintained in wiki (#470) per @ollyg: "the manual files were all converted into GitHub wiki pages" --- share/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/share/config.yml b/share/config.yml index c1d26ea4..7bff71ec 100644 --- a/share/config.yml +++ b/share/config.yml @@ -3,7 +3,9 @@ # DO NOT EDIT THIS FILE # # Overrides should go to ~/environments/deployment.yml -# See App::Netdisco::Manual::Configuration for explanations +# +# https://github.com/netdisco/netdisco/wiki/Configuration has +# in depth explanations about each setting. # ---------------- # GENERAL SETTINGS From f5622929e01a6578461639afb70e12ab72b5aca7 Mon Sep 17 00:00:00 2001 From: Christian Ramseyer Date: Fri, 28 Dec 2018 15:33:02 +0100 Subject: [PATCH 07/24] Provide stdin to IOSXR.pm when none available (#462) * See issue #461 for a detailed description --- lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm b/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm index 37d44e00..a3b1210c 100644 --- a/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm +++ b/lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm @@ -36,6 +36,11 @@ Returns a list of hashrefs in the format C<{ mac => MACADDR, ip => IPADDR }>. sub arpnip { my ($self, $hostlabel, $ssh, $args) = @_; + # IOSXR show commands seem to depend on an available STDIN + unless (-t STDIN){ + open STDIN, "<", "/dev/zero" or warn "Failed to fake stdin: $!"; + } + debug "$hostlabel $$ arpnip()"; my @data = $ssh->capture("show arp vrf all"); From b1ca1b968d6cacecea360a251bfd3ec31f9e7337 Mon Sep 17 00:00:00 2001 From: Christian Ramseyer Date: Fri, 28 Dec 2018 16:58:32 +0100 Subject: [PATCH 08/24] Improve netdisco-sshcollector CLI experience (#460) * Improve netdisco-sshcollector CLI experience * add -d * add -O debug option to toggle Net::OpenSSH::debug * add -w MCE::Loop max_workers * updated POD for all of the above * Fix -Q documentation * Specifying -Q multiple times has no effect, removed part talking about that. * Emit Net::OpenSSH connection failures as warnings --- bin/netdisco-sshcollector | 54 +++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 48d97306..04b66052 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -53,10 +53,13 @@ use Pod::Usage 'pod2usage'; use Getopt::Long; Getopt::Long::Configure ("bundling"); -my ($debug, $sqltrace) = (undef, 0); +my ($debug, $sqltrace, $device, $opensshdebug, $workers) = (undef, 0, undef, undef, "auto"); my $result = GetOptions( - 'debug|D' => \$debug, - 'sqltrace|Q+' => \$sqltrace, + 'debug|D' => \$debug, + 'sqltrace|Q' => \$sqltrace, + 'device|d=s' => \$device, + 'opensshdebug|O' => \$opensshdebug, + 'workers|w=i' => \$workers, ) or pod2usage( -msg => 'error: bad options', -verbose => 0, @@ -71,10 +74,11 @@ $ENV{DBIC_TRACE} ||= $sqltrace; # reconfigure logging to force console output Dancer::Logger->init('console', $CONFIG); -#this may be helpful with SSH issues: -#$Net::OpenSSH::debug = ~0; +if ($opensshdebug){ + $Net::OpenSSH::debug = ~0; +} -MCE::Loop::init { chunk_size => 1 }; +MCE::Loop::init { chunk_size => 1, max_workers => $workers }; my %stats; $stats{entry} = 0; @@ -83,6 +87,11 @@ exit main(); sub main { my @input = @{ setting('sshcollector') }; + if ($device){ + @input = grep{ ($_->{hostname} && $_->{hostname} eq $device) + || ($_->{ip} && $_->{ip} eq $device) } @input; + } + #one-line Fisher-Yates from https://www.perlmonks.org/index.pl?node=Array%20One-Liners my ($i,$j) = (0); @input[-$i,$j] = @input[$j,-$i] while $j = rand(@input - $i), ++$i < @input; @@ -108,7 +117,12 @@ sub main { ], ); - MCE->gather( process($hostlabel, $ssh, $host) ); + + if ($ssh->error){ + warning "WARNING: Couldn't connect to <$hostlabel> - " . $ssh->error; + }else{ + MCE->gather( process($hostlabel, $ssh, $host) ); + } } } \@input; @@ -173,7 +187,10 @@ full SNMP support ~/bin/localenv cpanm --notest Net::OpenSSH Expect # run manually, or add to cron: - ~/bin/netdisco-sshcollector [-DQ] + ~/bin/netdisco-sshcollector [-DQO] [-w ] + + # limit run to a single device defined in the config + ~/bin/netdisco-sshcollector [-DQO] [-w ] -d =head1 DESCRIPTION @@ -271,10 +288,7 @@ don't support command execution via ssh: The returned IP and MAC addresses should be in a format that the respective B and B datatypes in PostgreSQL can handle. -=head1 DEBUG LEVELS - -The flags "C<-DQ>" can be specified, multiple times, and enable the following -items in order: +=head1 COMMAND LINE OPTIONS =over 4 @@ -284,7 +298,21 @@ Netdisco debug log level =item C<-Q> -L trace enabled +L trace enabled. + +=item C<-O> + +L trace enabled + +=item C<-w> + +Set maximum parallel workers for L. The default is B. + +=item C<-d device> + +Only run for a single device. Takes an IP or hostname, must exactly match the value +in the config file. + =back From 08589ebf9683472d6e919f56d39dac03c894a28f Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Fri, 28 Dec 2018 17:00:13 +0100 Subject: [PATCH 09/24] workaround for #457 (#465) i'm not in anyway versed in javascript, but this hack fixes #457 for me. --- share/public/javascripts/portsort.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/public/javascripts/portsort.js b/share/public/javascripts/portsort.js index 2de54fc4..1a2d0c71 100644 --- a/share/public/javascripts/portsort.js +++ b/share/public/javascripts/portsort.js @@ -30,6 +30,8 @@ function portSort (a, b) { // octal regex ore = /^0/, // convert all to strings and trim() + if (typeof a == 'null') { a = ''; } + if (typeof b == 'null') { b = ''; } x = a.toString().replace(sre, '') || '', y = b.toString().replace(sre, '') || ''; From 86c38c94662aa0f83b2c299ffce86bf94440b2a3 Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Fri, 28 Dec 2018 17:01:40 +0100 Subject: [PATCH 10/24] make job queue list in webinterface configurable (#469) * add an option so you can define the amount of jobs returned in the webinterface jobq * and now the actually working version --- lib/App/Netdisco/JobQueue.pm | 5 +++-- lib/App/Netdisco/JobQueue/PostgreSQL.pm | 2 +- share/config.yml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/App/Netdisco/JobQueue.pm b/lib/App/Netdisco/JobQueue.pm index 733cf4bb..7e5448bd 100644 --- a/lib/App/Netdisco/JobQueue.pm +++ b/lib/App/Netdisco/JobQueue.pm @@ -59,8 +59,9 @@ Performs initialisation of the Job Queue backend. =head2 jq_log() -Returns a list of the most recent 50 jobs in the queue. Jobs are returned as -objects which implement the Netdisco job instance interface (see below). +Returns a list of the most recent jobs as defined in C from the +queue. Jobs are returned as objects which implement the Netdisco job instance +interface (see below). =head2 jq_userlog( $user ) diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index a16913f0..7cd3c9ff 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -273,7 +273,7 @@ sub jq_log { return schema('netdisco')->resultset('Admin')->search({}, { prefetch => 'target', order_by => { -desc => [qw/entered device action/] }, - rows => 50, + rows => (setting('jobs_qdepth') || 50), })->with_times->hri->all; } diff --git a/share/config.yml b/share/config.yml index 7bff71ec..c5d77fa9 100644 --- a/share/config.yml +++ b/share/config.yml @@ -324,6 +324,7 @@ workers: # 50 minutes jobs_stale_after: 3000 +jobs_qdepth: 50 dns: max_outstanding: 50 From 8aed7369256264fd5522817d39c20fae55970f49 Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Fri, 28 Dec 2018 17:08:40 +0100 Subject: [PATCH 11/24] ignore notpresent interfaces, with type filtering config item (set to [] to not filter). work done by postilion/nic bernstein, filter by me. also includes trailing space fix by nic. (#464) --- lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm | 9 ++++++++- share/config.yml | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm index e2b0651c..f2537ca7 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm @@ -181,6 +181,13 @@ register_worker({ phase => 'early', driver => 'snmp' }, sub { next; } + # Skip interfaces which are 'notPresent' and match the notpresent type filter + if (defined $i_up->{$entry} and defined $i_type->{$entry} and $i_up->{$entry} eq 'notPresent' and (scalar grep {$i_type->{$entry} =~ m/^$_$/} @{setting('ignore_notpresent_types') || []}) ) { + debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s) (config:ignore_notpresent_types)', + $device->ip, $entry, $port, $i_up->{$entry}; + next; + } + my $lc = $i_lastchange->{$entry} || 0; if (not $dev_uptime_wrapped and $lc > $dev_uptime) { info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting', @@ -263,7 +270,7 @@ sub _get_vrf_list { if ($vrf =~ /^\S+$/) { my $ctx_name = pack("C*",split(/\./,$idx)); $ctx_name =~ s/.*[^[:print:]]+//; - debug sprintf(' [%s] Discover VRF %s with SNMP Context %s', $device->ip, $vrf, $ctx_name); + debug sprintf(' [%s] Discover VRF %s with SNMP Context %s', $device->ip, $vrf, $ctx_name); push (@ok_vrfs, $ctx_name); } } diff --git a/share/config.yml b/share/config.yml index c5d77fa9..12d066ca 100644 --- a/share/config.yml +++ b/share/config.yml @@ -296,6 +296,10 @@ ignore_interfaces: - 'Ethernet(?:-| )QOS Packet Schedu?ler' - 'Ethernet(?:-| )WFP (?:802\.3|Native) MAC Layer Lightweight Filter' - 'ii\d\/\d\/\d+' +ignore_notpresent_types: + - 'ethernetCsmacd' + - 'tunnel' + - 'ieee8023adLag' ignore_private_nets: false reverse_sysname: false phone_capabilities: From 7f8db7b00d1e8fc5ec2415ed774ad703f2fa9023 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 16:43:08 +0000 Subject: [PATCH 12/24] build helpers for Mojave --- Build.PL | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Build.PL b/Build.PL index bbba12a4..54ce292e 100644 --- a/Build.PL +++ b/Build.PL @@ -11,6 +11,9 @@ Module::Build->new( 'Module::Build' => '0.42', }, build_requires => { + 'ExtUtils::Config' => '0', + 'ExtUtils::Helpers' => '0', + 'ExtUtils::InstallPaths' => '0', }, requires => { 'aliased' => '0', From b0f17db700ced620bfdaff7c76afb78a9ec0de33 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 16:50:22 +0000 Subject: [PATCH 13/24] Revert "workaround for #457 (#465)" This reverts commit 08589ebf9683472d6e919f56d39dac03c894a28f. --- share/public/javascripts/portsort.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/share/public/javascripts/portsort.js b/share/public/javascripts/portsort.js index 1a2d0c71..2de54fc4 100644 --- a/share/public/javascripts/portsort.js +++ b/share/public/javascripts/portsort.js @@ -30,8 +30,6 @@ function portSort (a, b) { // octal regex ore = /^0/, // convert all to strings and trim() - if (typeof a == 'null') { a = ''; } - if (typeof b == 'null') { b = ''; } x = a.toString().replace(sre, '') || '', y = b.toString().replace(sre, '') || ''; From 097bad6c481dc8582196b1652f7dec1b159dbc43 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 17:02:56 +0000 Subject: [PATCH 14/24] release 2.040000 --- Changes | 18 ++++++++++++++++++ META.json | 11 +++++++++-- META.yml | 7 +++++-- lib/App/Netdisco.pm | 2 +- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Changes b/Changes index d4efc08d..0ea69eee 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,21 @@ +2.040000 - 2018-12-28 + + [NEW FEATURES] + + * #464 ignore_notpresent_types config to ignore notPresent interfaces (inphobia, nic) + * #469 jobs_qdepth config to set length of job queue in web view (inphobia) + + [ENHANCEMENTS] + + * #459 additional interface type to ignore (inphobia) + * #460 several improvements to netdisco-sshcollector (rc9000) + + [BUG FIXES] + + * #456 fix POD error (manwar) + * #461 provide stdin to sshcollector (rc9000) + * #467 documentation fixes (inphobia) + 2.039033 - 2018-10-19 [BUG FIXES] diff --git a/META.json b/META.json index 21787b8a..ecf94c10 100644 --- a/META.json +++ b/META.json @@ -14,6 +14,13 @@ }, "name" : "App-Netdisco", "prereqs" : { + "build" : { + "requires" : { + "ExtUtils::Config" : "0", + "ExtUtils::Helpers" : "0", + "ExtUtils::InstallPaths" : "0" + } + }, "configure" : { "requires" : { "DBIx::Class" : "0.082810", @@ -110,7 +117,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.039033" + "version" : "2.040000" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -789,6 +796,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.039033", + "version" : "2.040000", "x_serialization_backend" : "JSON::PP version 2.97001" } diff --git a/META.yml b/META.yml index a0787865..ba8306f9 100644 --- a/META.yml +++ b/META.yml @@ -4,6 +4,9 @@ author: - 'Oliver Gorwits ' build_requires: Env::Path: '0' + ExtUtils::Config: '0' + ExtUtils::Helpers: '0' + ExtUtils::InstallPaths: '0' Test::File::ShareDir::Dist: '0' Test::More: '1.302083' configure_requires: @@ -19,7 +22,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.039033' + version: '2.040000' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -543,5 +546,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: '2.039033' +version: '2.040000' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 2b52e45a..72d74c93 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.039033'; +our $VERSION = '2.040000'; use App::Netdisco::Configuration; =head1 NAME From b445d237e48917ad3069d9cb865970dba8de754b Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 17:32:40 +0000 Subject: [PATCH 15/24] #471 OSPF Neighbor Discovery by router ID --- .../Worker/Plugin/Discover/Neighbors/Routed.pm | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm index 4e2f8760..a12d6b3d 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors/Routed.pm @@ -16,17 +16,18 @@ register_worker({ phase => 'main', driver => 'snmp' }, sub { my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) or return Status->defer("discover failed: could not SNMP connect to $device"); - my $ospf_peers = $snmp->ospf_peers || {}; - my $bgp_peers = $snmp->bgp_peer_addr || {}; - my $eigrp_peers = $snmp->eigrp_peers || {}; + my $ospf_peers = $snmp->ospf_peers || {}; + my $ospf_routers = $snmp->ospf_peer_id || {}; + my $bgp_peers = $snmp->bgp_peer_addr || {}; + my $eigrp_peers = $snmp->eigrp_peers || {}; return Status->info(" [$device] neigh - no BGP, OSPF, or EIGRP peers") unless ((scalar values %$ospf_peers) or (scalar values %$bgp_peers) or (scalar values %$eigrp_peers)); my $count = 0; - foreach my $ip ((values %$ospf_peers), (values %$bgp_peers), - (values %$eigrp_peers)) { + foreach my $ip ((values %$ospf_peers), (values %$ospf_routers), + (values %$bgp_peers), (values %$eigrp_peers)) { my $peer = get_device($ip); next if $peer->in_storage or not is_discoverable($peer); next if vars->{'queued'}->{$ip}; From b4e5dc7af7cdcb584921b051b6ab9451cffda96c Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 17:56:12 +0000 Subject: [PATCH 16/24] #422 Network map positions will not save (onlinehupe) --- share/views/ajax/device/netmap.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/views/ajax/device/netmap.tt b/share/views/ajax/device/netmap.tt index 233ce9f6..4a27fba8 100644 --- a/share/views/ajax/device/netmap.tt +++ b/share/views/ajax/device/netmap.tt @@ -216,7 +216,7 @@ function saveMapPositions() { graph.inspect().main.nodes.each(function(n) { n.fixed = true }); $.post( '[% uri_for('/ajax/data/device/netmappositions') %]' - ,$("#nd_vlan-entry, #nd_hgroup-select, #nd_lgroup-select, input[name='q'], input[name='mapshow']").serialize() + ,$("#nd_vlan-entry, #nd_hgroup-select, #nd_lgroup-select, #nq, input[name='mapshow']").serialize() + '&positions=' + JSON.stringify(graph.positions()) ); toastr.success('Saved map positions.'); From 5cc99cf20f5ba0e03f9ed576b3352587a1e20922 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 18:24:05 +0000 Subject: [PATCH 17/24] Update .travis.yml Send git commit emails to the mail list from Travis, instead of from GitHub (they are retiring the option for that). --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7992cb52..9740aefa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,3 +22,8 @@ notifications: on_failure: always channels: - 'chat.freenode.net#netdisco' + email: + on_success: always + on_failure: always + recipients: + - 'netdisco-ng-commit@lists.sourceforge.net' From e4dabe525ce1317d6685fea42ffd5f1e1c032668 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 18:34:29 +0000 Subject: [PATCH 18/24] Always notify from Travis to IRC and email on any code commit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9740aefa..af946ef6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ script: | ND_PHANTOMJS=/usr/bin/phantomjs ./Build test --test_files xt/ notifications: irc: - on_success: change + on_success: always on_failure: always channels: - 'chat.freenode.net#netdisco' From 6ed29f85ea372ec530cd7a01e30e5f3c2fd36b84 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Fri, 28 Dec 2018 22:56:51 +0000 Subject: [PATCH 19/24] update changes --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 0ea69eee..5ec56851 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,5 @@ +2.040001 - 2018-12-xx + 2.040000 - 2018-12-28 [NEW FEATURES] From 9022e3ee8b7b6c23954a84820ec1251e3dc0444e Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sat, 29 Dec 2018 13:26:54 +0000 Subject: [PATCH 20/24] updates to work with PhantomJS 2.x --- xt/js/run_qunit.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xt/js/run_qunit.js b/xt/js/run_qunit.js index c6d236f5..2412d3fa 100644 --- a/xt/js/run_qunit.js +++ b/xt/js/run_qunit.js @@ -1,3 +1,4 @@ +var system = require('system'); /** * Wait until the test condition is true or a timeout occurs. Useful for waiting * on a server response or for a ui change (fadeIn, etc.) to occur. @@ -34,7 +35,7 @@ function waitFor(testFx, onReady, timeOutMillis) { }; -if (phantom.args.length === 0 || phantom.args.length > 2) { +if (system.args.length === 0 || system.args.length > 2) { console.log('Usage: run-qunit.js URL'); phantom.exit(1); } @@ -46,7 +47,7 @@ page.onConsoleMessage = function(msg) { console.log(msg); }; -page.open(phantom.args[0], function(status){ +page.open(system.args[1], function(status){ if (status !== "success") { console.log("Unable to access network"); phantom.exit(1); From 5fb9d52278c66402c69060d09cfc71ea4d105b1e Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Sun, 30 Dec 2018 00:26:07 +0100 Subject: [PATCH 21/24] reference wiki, not nonexisting manual --- share/environments/deployment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/share/environments/deployment.yml b/share/environments/deployment.yml index c8e6fa1b..cd792907 100644 --- a/share/environments/deployment.yml +++ b/share/environments/deployment.yml @@ -2,7 +2,9 @@ # NETDISCO 2 CONFIGURATION FILE # # Settings in this file override share/config.yml -# See App::Netdisco::Manual::Configuration for more info. +# +# https://github.com/netdisco/netdisco/wiki/Configuration has +# in depth explanations about each setting. # ------------------ # ESSENTIAL SETTINGS From 485abbc9bd4d68e2c39e44ee6223dde81708d9ac Mon Sep 17 00:00:00 2001 From: nick n <39005454+inphobia@users.noreply.github.com> Date: Sun, 30 Dec 2018 07:52:27 +0100 Subject: [PATCH 22/24] trailing whitespace --- share/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/config.yml b/share/config.yml index 12d066ca..24444fd4 100644 --- a/share/config.yml +++ b/share/config.yml @@ -3,7 +3,7 @@ # DO NOT EDIT THIS FILE # # Overrides should go to ~/environments/deployment.yml -# +# # https://github.com/netdisco/netdisco/wiki/Configuration has # in depth explanations about each setting. From 22a3e8c9f6d76e74d2ee7ab92628adaec2842a55 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sun, 30 Dec 2018 10:52:40 +0000 Subject: [PATCH 23/24] release 2.040001 --- Build.PL | 2 +- Changes | 13 ++++++++++++- META.json | 6 +++--- META.yml | 6 +++--- lib/App/Netdisco.pm | 2 +- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Build.PL b/Build.PL index 54ce292e..6cf49a9a 100644 --- a/Build.PL +++ b/Build.PL @@ -71,7 +71,7 @@ Module::Build->new( 'Starman' => '0.4008', 'Storable' => '0', 'Sys::SigAction' => '0', - 'SNMP::Info' => '3.61', + 'SNMP::Info' => '3.64', 'SQL::Abstract' => '1.85', 'SQL::Translator' => '0.11024', 'Template' => '2.24', diff --git a/Changes b/Changes index 5ec56851..234420da 100644 --- a/Changes +++ b/Changes @@ -1,4 +1,15 @@ -2.040001 - 2018-12-xx +2.040001 - 2018-12-30 + + [ENHANCEMENTS] + + * #471 OSPF neighbor discovey will try Router ID as well as Peer ID + * update dependency to SNMP::Info 3.64 + + [BUG FIXES] + + * #422 fix for Network Map positions not saving (onlinehupe) + * fix portsort tests to work with PhantomJS 2.x + * some documentation fixes (inphobia) 2.040000 - 2018-12-28 diff --git a/META.json b/META.json index ecf94c10..2b47d72a 100644 --- a/META.json +++ b/META.json @@ -81,7 +81,7 @@ "Plack::Middleware::ReverseProxy" : "0.15", "Pod::Usage" : "0", "Role::Tiny" : "1.002005", - "SNMP::Info" : "3.61", + "SNMP::Info" : "3.64", "SQL::Abstract" : "1.85", "SQL::Translator" : "0.11024", "Scope::Guard" : "0", @@ -117,7 +117,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.040000" + "version" : "2.040001" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -796,6 +796,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.040000", + "version" : "2.040001", "x_serialization_backend" : "JSON::PP version 2.97001" } diff --git a/META.yml b/META.yml index ba8306f9..92de3fac 100644 --- a/META.yml +++ b/META.yml @@ -22,7 +22,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.040000' + version: '2.040001' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -516,7 +516,7 @@ requires: Plack::Middleware::ReverseProxy: '0.15' Pod::Usage: '0' Role::Tiny: '1.002005' - SNMP::Info: '3.61' + SNMP::Info: '3.64' SQL::Abstract: '1.85' SQL::Translator: '0.11024' Scope::Guard: '0' @@ -546,5 +546,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: '2.040000' +version: '2.040001' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 72d74c93..36f1fd1d 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.040000'; +our $VERSION = '2.040001'; use App::Netdisco::Configuration; =head1 NAME From df769c97203f8f414af30c643aad922d7904b88d Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sun, 30 Dec 2018 10:57:54 +0000 Subject: [PATCH 24/24] release 2.040002 --- Changes | 6 ++++++ META.json | 4 ++-- META.yml | 4 ++-- lib/App/Netdisco.pm | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Changes b/Changes index 234420da..358c0e9e 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,9 @@ +2.040002 - 2018-12-30 + + [BUG FIXES] + + * correction for git sync + 2.040001 - 2018-12-30 [ENHANCEMENTS] diff --git a/META.json b/META.json index 2b47d72a..e58fa24c 100644 --- a/META.json +++ b/META.json @@ -117,7 +117,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.040001" + "version" : "2.040002" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -796,6 +796,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.040001", + "version" : "2.040002", "x_serialization_backend" : "JSON::PP version 2.97001" } diff --git a/META.yml b/META.yml index 92de3fac..190948b6 100644 --- a/META.yml +++ b/META.yml @@ -22,7 +22,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.040001' + version: '2.040002' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -546,5 +546,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: '2.040001' +version: '2.040002' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 36f1fd1d..7904c0b0 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.040001'; +our $VERSION = '2.040002'; use App::Netdisco::Configuration; =head1 NAME