From d19b2fac934d1f28da30e91fe627f2e860cb8faa Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sun, 8 Apr 2018 13:22:52 +0100 Subject: [PATCH 01/13] save a device_port lookup --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 86de75e3..613e402a 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -238,7 +238,6 @@ sub store_neighbors { # OK, remote IP seems sane $remote_ip = $r_netaddr->addr; - my $peer_device = get_device($remote_ip); # what we came here to do.... discover the neighbor debug sprintf ' [%s] neigh - %s with ID [%s] on %s', @@ -246,6 +245,9 @@ sub store_neighbors { push @to_discover, [$remote_ip, $remote_type, $remote_id]; $remote_port = $c_port->{$entry}; + my $peer_device = get_device($remote_ip); + my $peer_port = undef; + if (defined $remote_port) { # clean weird characters $remote_port =~ s/[^\d\s\/\.,()\w:-]+//gi; @@ -253,7 +255,7 @@ sub store_neighbors { # attempt to resolve port name when it is given wrong # https://github.com/netdisco/netdisco/issues/380 if ($peer_device and $peer_device->in_storage) { - my $peer_port = schema('netdisco')->resultset('DevicePort')->search({ + $peer_port = schema('netdisco')->resultset('DevicePort')->search({ ip => $peer_device->ip, port => [ {'=', $remote_port}, @@ -291,11 +293,6 @@ sub store_neighbors { port => $portrow->slave_of }); - my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ - ip => $peer_device->ip, - port => $portrow->remote_port, - }); - if ($master and not ($portrow->is_master or defined $master->slave_of)) { $master->update({ remote_ip => ($peer_device->ip || $remote_ip), From 8bcfaf12d0511ed084db5116896de4377f25663b Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Mon, 9 Apr 2018 22:58:23 +0100 Subject: [PATCH 02/13] better way to construct short interface name --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 613e402a..ed5c3818 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -312,8 +312,8 @@ sub normalize_port { my $port = shift or return ''; my ($start, $end) = ('', ''); if ($port =~ m/^([a-z]{2})/i) { $start = $1 } - ($end = $port) =~ s/^\D+//; - return (quotemeta($start) .'%'. quotemeta($end)); + if ($port =~ m/[a-z]([^a-z]+)$/i) { $end = $1 } + return (($start && $end) ? "${start}${end}" : $port); } # take data from the topology table and update remote_ip and remote_port From 9d48c4a025f7b0ce61f8c24025b948df53c26f44 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Mon, 9 Apr 2018 23:05:17 +0100 Subject: [PATCH 03/13] work for shortening or lengthening ports --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index ed5c3818..5e15a56f 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -259,7 +259,8 @@ sub store_neighbors { ip => $peer_device->ip, port => [ {'=', $remote_port}, - {'-ilike', normalize_port($remote_port)}, + {'-ilike', long_port($remote_port)}, + {'-ilike', short_port($remote_port)}, {'-ilike', ('%'.quotemeta($remote_port))}, ], }, { rows => 1 })->single(); @@ -308,7 +309,13 @@ sub store_neighbors { return @to_discover; } -sub normalize_port { +sub long_port { + my $port = shift or return ''; + $port =~ s/^([a-z]{2})([^a-z].+)$/$1%$2/i; + return $port; +} + +sub short_port { my $port = shift or return ''; my ($start, $end) = ('', ''); if ($port =~ m/^([a-z]{2})/i) { $start = $1 } From 9339c1a2cead264c35eb5dd9dc078ba7b19eec26 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Mon, 9 Apr 2018 23:15:58 +0100 Subject: [PATCH 04/13] no need for quotemeta --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 5e15a56f..532443c2 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -261,7 +261,7 @@ sub store_neighbors { {'=', $remote_port}, {'-ilike', long_port($remote_port)}, {'-ilike', short_port($remote_port)}, - {'-ilike', ('%'.quotemeta($remote_port))}, + {'-ilike', ('%'. $remote_port)}, ], }, { rows => 1 })->single(); if ($peer_port and ($peer_port->port ne $remote_port)) { @@ -311,7 +311,7 @@ sub store_neighbors { sub long_port { my $port = shift or return ''; - $port =~ s/^([a-z]{2})([^a-z].+)$/$1%$2/i; + $port =~ s/^([a-z]{2})(\d.+)$/$1%$2/i; return $port; } From 52e14aeeebde21bdf84efe5c77e6daf407140fc6 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 20:58:02 +0100 Subject: [PATCH 05/13] Revert "no need for quotemeta" This reverts commit 9339c1a2cead264c35eb5dd9dc078ba7b19eec26. --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 532443c2..5e15a56f 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -261,7 +261,7 @@ sub store_neighbors { {'=', $remote_port}, {'-ilike', long_port($remote_port)}, {'-ilike', short_port($remote_port)}, - {'-ilike', ('%'. $remote_port)}, + {'-ilike', ('%'.quotemeta($remote_port))}, ], }, { rows => 1 })->single(); if ($peer_port and ($peer_port->port ne $remote_port)) { @@ -311,7 +311,7 @@ sub store_neighbors { sub long_port { my $port = shift or return ''; - $port =~ s/^([a-z]{2})(\d.+)$/$1%$2/i; + $port =~ s/^([a-z]{2})([^a-z].+)$/$1%$2/i; return $port; } From 00f3f2a796fedf715a4d86792d68f809293886ae Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 20:58:05 +0100 Subject: [PATCH 06/13] Revert "work for shortening or lengthening ports" This reverts commit 9d48c4a025f7b0ce61f8c24025b948df53c26f44. --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 5e15a56f..ed5c3818 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -259,8 +259,7 @@ sub store_neighbors { ip => $peer_device->ip, port => [ {'=', $remote_port}, - {'-ilike', long_port($remote_port)}, - {'-ilike', short_port($remote_port)}, + {'-ilike', normalize_port($remote_port)}, {'-ilike', ('%'.quotemeta($remote_port))}, ], }, { rows => 1 })->single(); @@ -309,13 +308,7 @@ sub store_neighbors { return @to_discover; } -sub long_port { - my $port = shift or return ''; - $port =~ s/^([a-z]{2})([^a-z].+)$/$1%$2/i; - return $port; -} - -sub short_port { +sub normalize_port { my $port = shift or return ''; my ($start, $end) = ('', ''); if ($port =~ m/^([a-z]{2})/i) { $start = $1 } From 67ee1808b6f969b52bfda0834f755b3b3f410de6 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 20:58:06 +0100 Subject: [PATCH 07/13] Revert "better way to construct short interface name" This reverts commit 8bcfaf12d0511ed084db5116896de4377f25663b. --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index ed5c3818..613e402a 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -312,8 +312,8 @@ sub normalize_port { my $port = shift or return ''; my ($start, $end) = ('', ''); if ($port =~ m/^([a-z]{2})/i) { $start = $1 } - if ($port =~ m/[a-z]([^a-z]+)$/i) { $end = $1 } - return (($start && $end) ? "${start}${end}" : $port); + ($end = $port) =~ s/^\D+//; + return (quotemeta($start) .'%'. quotemeta($end)); } # take data from the topology table and update remote_ip and remote_port From 08a5652ae079fb7998151d3d7dcfbe185814cbec Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 20:58:07 +0100 Subject: [PATCH 08/13] Revert "save a device_port lookup" This reverts commit d19b2fac934d1f28da30e91fe627f2e860cb8faa. --- lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 613e402a..86de75e3 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -238,6 +238,7 @@ sub store_neighbors { # OK, remote IP seems sane $remote_ip = $r_netaddr->addr; + my $peer_device = get_device($remote_ip); # what we came here to do.... discover the neighbor debug sprintf ' [%s] neigh - %s with ID [%s] on %s', @@ -245,9 +246,6 @@ sub store_neighbors { push @to_discover, [$remote_ip, $remote_type, $remote_id]; $remote_port = $c_port->{$entry}; - my $peer_device = get_device($remote_ip); - my $peer_port = undef; - if (defined $remote_port) { # clean weird characters $remote_port =~ s/[^\d\s\/\.,()\w:-]+//gi; @@ -255,7 +253,7 @@ sub store_neighbors { # attempt to resolve port name when it is given wrong # https://github.com/netdisco/netdisco/issues/380 if ($peer_device and $peer_device->in_storage) { - $peer_port = schema('netdisco')->resultset('DevicePort')->search({ + my $peer_port = schema('netdisco')->resultset('DevicePort')->search({ ip => $peer_device->ip, port => [ {'=', $remote_port}, @@ -293,6 +291,11 @@ sub store_neighbors { port => $portrow->slave_of }); + my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ + ip => $peer_device->ip, + port => $portrow->remote_port, + }); + if ($master and not ($portrow->is_master or defined $master->slave_of)) { $master->update({ remote_ip => ($peer_device->ip || $remote_ip), From 454ededa4671fc3ac749dbd2ea2e28b1ae3c3425 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 21:03:10 +0100 Subject: [PATCH 09/13] Revert "#380 attempt to resolve port name when it is given wrong" This reverts commit f4a143612b93594ddc37480727f2e0e317d122b2. --- .../Worker/Plugin/Discover/Neighbors.pm | 43 +++++-------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm index 86de75e3..d34015e3 100644 --- a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -238,7 +238,6 @@ sub store_neighbors { # OK, remote IP seems sane $remote_ip = $r_netaddr->addr; - my $peer_device = get_device($remote_ip); # what we came here to do.... discover the neighbor debug sprintf ' [%s] neigh - %s with ID [%s] on %s', @@ -249,24 +248,6 @@ sub store_neighbors { if (defined $remote_port) { # clean weird characters $remote_port =~ s/[^\d\s\/\.,()\w:-]+//gi; - - # attempt to resolve port name when it is given wrong - # https://github.com/netdisco/netdisco/issues/380 - if ($peer_device and $peer_device->in_storage) { - my $peer_port = schema('netdisco')->resultset('DevicePort')->search({ - ip => $peer_device->ip, - port => [ - {'=', $remote_port}, - {'-ilike', normalize_port($remote_port)}, - {'-ilike', ('%'.quotemeta($remote_port))}, - ], - }, { rows => 1 })->single(); - if ($peer_port and ($peer_port->port ne $remote_port)) { - info sprintf ' [%s] neigh - changing remote port on %s from %s to %s', - $device->ip, $port, $remote_port, $peer_port->port; - $remote_port = $peer_port->port; - } - } } else { info sprintf ' [%s] neigh - no remote port found for port %s at %s', @@ -285,18 +266,22 @@ sub store_neighbors { # update master of our aggregate to be a neighbor of # the master on our peer device (a lot of iffs to get there...). # & cannot use ->neighbor prefetch because this is the port insert! - if ($peer_device and $peer_device->in_storage and defined $portrow->slave_of) { + if (defined $portrow->slave_of) { + + my $peer_device = get_device($remote_ip); my $master = schema('netdisco')->resultset('DevicePort')->single({ ip => $device->ip, port => $portrow->slave_of }); - my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ - ip => $peer_device->ip, - port => $portrow->remote_port, - }); + if ($peer_device and $peer_device->in_storage and $master + and not ($portrow->is_master or defined $master->slave_of)) { + + my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ + ip => $peer_device->ip, + port => $portrow->remote_port, + }); - if ($master and not ($portrow->is_master or defined $master->slave_of)) { $master->update({ remote_ip => ($peer_device->ip || $remote_ip), remote_port => ($peer_port ? $peer_port->slave_of : undef ), @@ -311,14 +296,6 @@ sub store_neighbors { return @to_discover; } -sub normalize_port { - my $port = shift or return ''; - my ($start, $end) = ('', ''); - if ($port =~ m/^([a-z]{2})/i) { $start = $1 } - ($end = $port) =~ s/^\D+//; - return (quotemeta($start) .'%'. quotemeta($end)); -} - # take data from the topology table and update remote_ip and remote_port # in the devices table. only use root_ips and skip any bad topo entries. sub set_manual_topology { From bebaaba6267d958e9f13ec1bf94601a2b413993a Mon Sep 17 00:00:00 2001 From: Brian De Wolf Date: Tue, 10 Apr 2018 13:12:20 -0700 Subject: [PATCH 10/13] fix sshcollector errors when devices are empty (#391) While setting up an sshcollector, I found that the script doesn't handle an arpless device very gracefully. The log looks like: [46567] 2018-04-10 16:59:08 warn WARNING: no entries received from Can't use string ("1") as an ARRAY ref while "strict refs" in use at perl5/bin/netdisco-sshcollector line 114, <__ANONIO__> line 3. This is fixed by making the process function always return the expected structure, even if it's empty. After this error, the following errors showed up: [53232] 2018-04-10 17:06:15 warn WARNING: no entries received from [53210] 2018-04-10 17:06:15 info [134.71.0.126] arpnip - retrieved 0 entries Use of uninitialized value in sprintf at perl5/bin/netdisco-sshcollector line 119, <__ANONIO__> line 3. [53210] 2018-04-10 17:06:15 info arpnip - processed ARP Cache entries from 1 devices This is because $stats{entry} only becomes an integer by being incremented, so if there aren't any entries, it never exists. This is fixed by starting it at 0. --- bin/netdisco-sshcollector | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index ccf12357..d77932a1 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -76,6 +76,7 @@ Dancer::Logger->init('console', $CONFIG); MCE::Loop::init { chunk_size => 1 }; my %stats; +$stats{entry} = 0; exit main(); @@ -130,13 +131,11 @@ sub process { my $arpentries = [ $device->arpnip($hostlabel, $ssh, $args) ]; # debug p $arpentries; - if (scalar @$arpentries) { - hostnames_resolve_async($arpentries); - return [$hostlabel, $arpentries]; - } - else { + if (not scalar @$arpentries) { warning "WARNING: no entries received from <$hostlabel>"; } + hostnames_resolve_async($arpentries); + return [$hostlabel, $arpentries]; } sub store_arpentries { From 6ee946cca6f134eaad4278e94b85e1826dbc179e Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 21:13:48 +0100 Subject: [PATCH 11/13] #389 build/upgrade issues (EL6) --- Build.PL | 1 + 1 file changed, 1 insertion(+) diff --git a/Build.PL b/Build.PL index c4bdb69d..b7ae527a 100644 --- a/Build.PL +++ b/Build.PL @@ -53,6 +53,7 @@ Module::Build->new( 'NetAddr::MAC' => '0.93', 'NetAddr::IP' => '4.068', 'Opcode' => '1.07', + 'Package::DeprecationManager' => '0', 'Path::Class' => '0.32', 'Plack' => '1.0023', 'Plack::Handler::Twiggy' => '0', From 11e49671fdfd7b0bfab50935fe7b4556cea972bf Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 21:22:06 +0100 Subject: [PATCH 12/13] #388 searching for 0.x.x.x returns Internal Server Error (C. Neuhaus) --- lib/App/Netdisco/Web/Plugin/Search/Node.pm | 1 + lib/App/Netdisco/Web/Search.pm | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/App/Netdisco/Web/Plugin/Search/Node.pm b/lib/App/Netdisco/Web/Plugin/Search/Node.pm index f369ad99..45394e2c 100644 --- a/lib/App/Netdisco/Web/Plugin/Search/Node.pm +++ b/lib/App/Netdisco/Web/Plugin/Search/Node.pm @@ -23,6 +23,7 @@ ajax '/ajax/content/search/node' => require_login sub { my ( $start, $end ) = param('daterange') =~ m/(\d+-\d+-\d+)/gmx; my $mac = NetAddr::MAC->new(mac => $node); + undef $mac if ($mac and $mac->as_ieee and ($mac->as_ieee eq '00:00:00:00')); my @active = (param('archived') ? () : (-bool => 'active')); my (@times, @wifitimes, @porttimes); diff --git a/lib/App/Netdisco/Web/Search.pm b/lib/App/Netdisco/Web/Search.pm index c1684f64..2d2cb01d 100644 --- a/lib/App/Netdisco/Web/Search.pm +++ b/lib/App/Netdisco/Web/Search.pm @@ -39,6 +39,7 @@ get '/search' => require_login sub { my $nd = $s->resultset('Device')->search_fuzzy($q); my ($likeval, $likeclause) = sql_match($q); my $mac = NetAddr::MAC->new($q); + undef $mac if ($mac and $mac->as_ieee and ($mac->as_ieee eq '00:00:00:00')); if ($nd and $nd->count) { if ($nd->count == 1) { From 8838f262c1eb2d66c01d9db6b7b5d1e57e9845e9 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Tue, 10 Apr 2018 21:25:16 +0100 Subject: [PATCH 13/13] update changes --- Changes | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Changes b/Changes index f4eff19d..ab1c617e 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,12 @@ +2.039021 - 2018-04- + + [BUG FIXES] + + * #389 build/upgrade issues (EL6) + * #390 cannot take logarithm of zero (C. Stromsoe) + * protect against undef mac (l.e. ferguson) + * do not include logical aggregate masters in netmap/speed calc + 2.039020 - 2018-03-26 [ENHANCEMENTS]