From fc4bfa862800da93bfd43e08ff8ed161c19fea78 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 21:42:01 +0000 Subject: [PATCH 1/9] fix sshcollector --- bin/netdisco-sshcollector | 3 +- lib/App/Netdisco/Util/Node.pm | 51 ++++++++++++++++++ .../Netdisco/Worker/Plugin/Arpnip/Nodes.pm | 53 +------------------ 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/bin/netdisco-sshcollector b/bin/netdisco-sshcollector index 8792d4c6..ccf12357 100755 --- a/bin/netdisco-sshcollector +++ b/bin/netdisco-sshcollector @@ -40,8 +40,7 @@ BEGIN { } use App::Netdisco; -use App::Netdisco::Core::Arpnip 'store_arp'; -use App::Netdisco::Util::Node 'check_mac'; +use App::Netdisco::Util::Node qw/check_mac store_arp/; use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; use Dancer ':script'; diff --git a/lib/App/Netdisco/Util/Node.pm b/lib/App/Netdisco/Util/Node.pm index d098dda0..ca6d4eed 100644 --- a/lib/App/Netdisco/Util/Node.pm +++ b/lib/App/Netdisco/Util/Node.pm @@ -11,6 +11,7 @@ our @EXPORT = (); our @EXPORT_OK = qw/ check_mac is_nbtstatable + store_arp /; our %EXPORT_TAGS = (all => \@EXPORT_OK); @@ -141,4 +142,54 @@ sub is_nbtstatable { return 1; } +=head2 store_arp( \%host, $now? ) + +Stores a new entry to the C table with the given MAC, IP (v4 or v6) +and DNS host name. Host details are provided in a Hash ref: + + { + ip => '192.0.2.1', + node => '00:11:22:33:44:55', + dns => 'myhost.example.com', + } + +The C entry is optional. The update will mark old entries for this IP as +no longer C. + +Optionally a literal string can be passed in the second argument for the +C timestamp, otherwise the current timestamp (C) is used. + +=cut + +sub store_arp { + my ($hash_ref, $now) = @_; + $now ||= 'now()'; + my $ip = $hash_ref->{'ip'}; + my $mac = NetAddr::MAC->new($hash_ref->{'node'}); + my $name = $hash_ref->{'dns'}; + + return if !defined $mac or $mac->errstr; + + schema('netdisco')->txn_do(sub { + my $current = schema('netdisco')->resultset('NodeIp') + ->search( + { ip => $ip, -bool => 'active'}, + { columns => [qw/mac ip/] })->update({active => \'false'}); + + schema('netdisco')->resultset('NodeIp') + ->update_or_create( + { + mac => $mac->as_ieee, + ip => $ip, + dns => $name, + active => \'true', + time_last => \$now, + }, + { + key => 'primary', + for => 'update', + }); + }); +} + 1; diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm index 4c5e82a4..a9620376 100644 --- a/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm @@ -5,11 +5,10 @@ use App::Netdisco::Worker::Plugin; use aliased 'App::Netdisco::Worker::Status'; use App::Netdisco::Transport::SNMP (); -use App::Netdisco::Util::Node 'check_mac'; +use App::Netdisco::Util::Node qw/check_mac store_arp/; use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; use Dancer::Plugin::DBIC 'schema'; use Time::HiRes 'gettimeofday'; -use NetAddr::MAC (); register_worker({ phase => 'main', driver => 'snmp' }, sub { my ($job, $workerconf) = @_; @@ -64,54 +63,4 @@ sub get_arps { return $resolved_ips; } -=head2 store_arp( \%host, $now? ) - -Stores a new entry to the C table with the given MAC, IP (v4 or v6) -and DNS host name. Host details are provided in a Hash ref: - - { - ip => '192.0.2.1', - node => '00:11:22:33:44:55', - dns => 'myhost.example.com', - } - -The C entry is optional. The update will mark old entries for this IP as -no longer C. - -Optionally a literal string can be passed in the second argument for the -C timestamp, otherwise the current timestamp (C) is used. - -=cut - -sub store_arp { - my ($hash_ref, $now) = @_; - $now ||= 'now()'; - my $ip = $hash_ref->{'ip'}; - my $mac = NetAddr::MAC->new($hash_ref->{'node'}); - my $name = $hash_ref->{'dns'}; - - return if !defined $mac or $mac->errstr; - - schema('netdisco')->txn_do(sub { - my $current = schema('netdisco')->resultset('NodeIp') - ->search( - { ip => $ip, -bool => 'active'}, - { columns => [qw/mac ip/] })->update({active => \'false'}); - - schema('netdisco')->resultset('NodeIp') - ->update_or_create( - { - mac => $mac->as_ieee, - ip => $ip, - dns => $name, - active => \'true', - time_last => \$now, - }, - { - key => 'primary', - for => 'update', - }); - }); -} - true; From f4889a19ffdf117a17215f4499dc5f53626bcf47 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 21:47:19 +0000 Subject: [PATCH 2/9] tweak comments in deploy file --- share/environments/deployment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/share/environments/deployment.yml b/share/environments/deployment.yml index 286749e6..c8e6fa1b 100644 --- a/share/environments/deployment.yml +++ b/share/environments/deployment.yml @@ -18,11 +18,6 @@ database: # RECOMMENDED SETTINGS # -------------------- -# will be stripped from fqdn when displayed in the web UI -# also, do not forget the leading dot. -# ``````````````````````````````````````````````````````` -#domain_suffix: '.example.com' - # SNMP community string(s) # ```````````````````````` device_auth: @@ -35,6 +30,11 @@ device_auth: # read: false # write: true +# will be stripped from fqdn when displayed in the web UI +# also, do not forget the leading dot. +# ``````````````````````````````````````````````````````` +#domain_suffix: '.example.com' + # ¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸ # ¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸.·´¯`·.¸¸ From fe574183e553f5fc28d68b7ad346a4de517a82ff Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 21:48:27 +0000 Subject: [PATCH 3/9] release 2.037000 --- Changes | 3 ++- META.json | 10 ++++++---- META.yml | 8 +++++--- lib/App/Netdisco.pm | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Changes b/Changes index 1abf3d2b..19324fb9 100644 --- a/Changes +++ b/Changes @@ -1,4 +1,4 @@ -2.036012_003 - EXPERIMENTAL RELEASE +2.037000 - 2017-12-14 [NEW FEATURES] @@ -11,6 +11,7 @@ * Deduplicate neighbors based on lldpRemChassisId * Scheduler config does not need to be uncommented * More efficient polling of the job queue + * Better Port search options and results presentation [BUG FIXES] diff --git a/META.json b/META.json index 86d849a9..1e99464e 100644 --- a/META.json +++ b/META.json @@ -78,6 +78,7 @@ "Sereal" : "0", "Socket6" : "0.23", "Starman" : "0.4008", + "Storable" : "0", "Template" : "2.24", "Template::Plugin::CSV" : "0.04", "Template::Plugin::Number::Format" : "1.02", @@ -85,7 +86,8 @@ "Term::UI" : "0", "Try::Tiny" : "0", "URI" : "0", - "URL::Encode" : "0.01", + "URL::Encode" : "0.03", + "URL::Encode::XS" : "0", "YAML" : "0.84", "YAML::XS" : "0.41", "aliased" : "0", @@ -103,7 +105,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.036012_003" + "version" : "2.037000" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -740,7 +742,7 @@ "file" : "lib/Dancer/Template/NetdiscoTemplateToolkit.pm" } }, - "release_status" : "testing", + "release_status" : "stable", "resources" : { "bugtracker" : { "web" : "https://github.com/netdisco/netdisco/issues" @@ -755,6 +757,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.036012_003", + "version" : "2.037000", "x_serialization_backend" : "JSON::PP version 2.94" } diff --git a/META.yml b/META.yml index 8a6291ec..f79688db 100644 --- a/META.yml +++ b/META.yml @@ -18,7 +18,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: 2.036012_003 + version: '2.037000' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -498,6 +498,7 @@ requires: Sereal: '0' Socket6: '0.23' Starman: '0.4008' + Storable: '0' Template: '2.24' Template::Plugin::CSV: '0.04' Template::Plugin::Number::Format: '1.02' @@ -505,7 +506,8 @@ requires: Term::UI: '0' Try::Tiny: '0' URI: '0' - URL::Encode: '0.01' + URL::Encode: '0.03' + URL::Encode::XS: '0' YAML: '0.84' YAML::XS: '0.41' aliased: '0' @@ -518,5 +520,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: 2.036012_003 +version: '2.037000' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index fa5a515f..13f577d5 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.036012_003'; +our $VERSION = '2.037000'; use App::Netdisco::Configuration; =head1 NAME From 242148acdbb890277236ec910d0a19afabc6f4db Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 21:56:21 +0000 Subject: [PATCH 4/9] Also update stats on running netdisco-deploy --- Changes | 6 ++++++ bin/netdisco-deploy | 5 +++++ lib/App/Netdisco.pm | 4 +--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Changes b/Changes index 19324fb9..718087b5 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,9 @@ +2.037000 - 2017-12-14 + + [ENHANCEMENTS] + + * Also update stats on running netdisco-deploy + 2.037000 - 2017-12-14 [NEW FEATURES] diff --git a/bin/netdisco-deploy b/bin/netdisco-deploy index bb34967e..e893198a 100755 --- a/bin/netdisco-deploy +++ b/bin/netdisco-deploy @@ -139,6 +139,11 @@ sub deploy_db { say 'DB schema update complete.'; print color 'reset'; + system 'netdisco-do stats >/dev/null 2>&1'; + print color 'bold blue'; + say 'Statistics updated.'; + print color 'reset'; + if (not setting('safe_password_store')) { say ''; print color 'bold red'; diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 13f577d5..43a7b352 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -219,13 +219,11 @@ Then the process below should be run for each installation: # apply database schema updates, update MIBs and Vendor MACs ~/bin/netdisco-deploy - ~/bin/netdisco-do stats # restart web service (if you run it) ~/bin/netdisco-web restart - # stop old daemon and start new backend worker (wherever you run them) - ~/bin/netdisco-daemon stop + # restart the backend workers (wherever you run them) ~/bin/netdisco-backend restart =head1 Tips and Tricks From 821699508370b381e8d44d653917a42989aac3bd Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 21:57:14 +0000 Subject: [PATCH 5/9] release 2.037001 --- Changes | 2 +- META.json | 4 ++-- META.yml | 4 ++-- lib/App/Netdisco.pm | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Changes b/Changes index 718087b5..5e6e2125 100644 --- a/Changes +++ b/Changes @@ -1,4 +1,4 @@ -2.037000 - 2017-12-14 +2.037001 - 2017-12-14 [ENHANCEMENTS] diff --git a/META.json b/META.json index 1e99464e..21e1168c 100644 --- a/META.json +++ b/META.json @@ -105,7 +105,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.037000" + "version" : "2.037001" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -757,6 +757,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.037000", + "version" : "2.037001", "x_serialization_backend" : "JSON::PP version 2.94" } diff --git a/META.yml b/META.yml index f79688db..a53f9893 100644 --- a/META.yml +++ b/META.yml @@ -18,7 +18,7 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.037000' + version: '2.037001' App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: @@ -520,5 +520,5 @@ resources: homepage: http://netdisco.org/ license: http://opensource.org/licenses/bsd-license.php repository: https://github.com/netdisco/netdisco -version: '2.037000' +version: '2.037001' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index 43a7b352..6443f2f7 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.037000'; +our $VERSION = '2.037001'; use App::Netdisco::Configuration; =head1 NAME From 0ed978d3fc4362d54d39eb8327a50d5a3e92afa7 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 22:02:53 +0000 Subject: [PATCH 6/9] improve netdisco-do docs --- bin/netdisco-do | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/netdisco-do b/bin/netdisco-do index 859de29d..daf3ca92 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -170,11 +170,19 @@ The C<-d> option will accept a hostname (that can be resolved to an IP with DNS), an IP address, or IP prefix (subnets in CIDR format). It can be any interface on the device known to Netdisco. +=head1 ACTIONS + Note that some jobs (C, C, C, C) simply add entries to the Netdisco job queue for other jobs, so won't seem to do much when you trigger them. -=head1 ACTIONS +For any action, if you wish to run one of its individual worker stages, then +pass C as the first argument to C, for example +C. + +Any action taking a C parameter can be passed either a hostname or IP, +or an IP prefix (subnet) which will cause C to run the action for +all addresses in that range. =head2 discover @@ -184,7 +192,7 @@ Run a discover on the device (specified with C<-d>). =head2 discoverall -Run a discover for all known devices. +Queue a discover for all known devices. =head2 macsuck @@ -194,7 +202,7 @@ Run a macsuck on the device (specified with C<-d>). =head2 macwalk -Run a macsuck for all known devices. +Queue a macsuck for all known devices. =head2 arpnip @@ -204,7 +212,7 @@ Run an arpnip on the device (specified with C<-d>). =head2 arpwalk -Run an arpnip for all known devices. +Queue an arpnip for all known devices. =head2 delete @@ -236,7 +244,7 @@ Run an nbtstat on the node (specified with C<-d>). =head2 nbtwalk -Run an nbtstat for all known nodes. +Queue an nbtstat for all known nodes. =head2 expire From bab7fab50ac378865237d14ff131158ac021aada Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Thu, 14 Dec 2017 22:04:54 +0000 Subject: [PATCH 7/9] improve netdisco-do docs --- bin/netdisco-do | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/netdisco-do b/bin/netdisco-do index daf3ca92..ed84659a 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -166,10 +166,6 @@ netdisco-do - Run any Netdisco job from the command-line. This program allows you to run any Netdisco poller job from the command-line. -The C<-d> option will accept a hostname (that can be resolved to an IP with -DNS), an IP address, or IP prefix (subnets in CIDR format). It can be any -interface on the device known to Netdisco. - =head1 ACTIONS Note that some jobs (C, C, C, C) @@ -180,9 +176,9 @@ For any action, if you wish to run one of its individual worker stages, then pass C as the first argument to C, for example C. -Any action taking a C parameter can be passed either a hostname or IP, -or an IP prefix (subnet) which will cause C to run the action for -all addresses in that range. +Any action taking a C parameter can be passed either a hostname or IP +of any interface on a known or unknown device, or an IP prefix (subnet) which +will cause C to run the action for all addresses in that range. =head2 discover From 55758964a09482af97351beafe80b89c0a3f2053 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sat, 16 Dec 2017 10:48:42 +0000 Subject: [PATCH 8/9] Include Connected Device ID in Device Ports sidebar cookie --- Changes | 7 +++++++ share/config.yml | 1 + 2 files changed, 8 insertions(+) diff --git a/Changes b/Changes index 5e6e2125..12dd20ba 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,10 @@ +2.037002 - 2017-12-14 + + [ENHANCEMENTS] + + * Include Connected Device ID in Device Ports sidebar cookie + * Improve netdisco-do docs + 2.037001 - 2017-12-14 [ENHANCEMENTS] diff --git a/share/config.yml b/share/config.yml index aba6a8e2..d79ab2af 100644 --- a/share/config.yml +++ b/share/config.yml @@ -131,6 +131,7 @@ sidebar_defaults: age_num: { default: 3 } age_unit: { default: months } mac_format: { default: IEEE } + neigh_id: { default: null } report_moduleinventory: fruonly: {default: checked } matchall: {default: checked } From c8806beaca93fad6b0e4516e4bb39a80f1b6e727 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sat, 16 Dec 2017 10:51:28 +0000 Subject: [PATCH 9/9] cleaner way to update stats after deploy --- bin/netdisco-deploy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/netdisco-deploy b/bin/netdisco-deploy index e893198a..e7106907 100755 --- a/bin/netdisco-deploy +++ b/bin/netdisco-deploy @@ -40,6 +40,7 @@ use App::Netdisco; use Dancer ':script'; use Dancer::Plugin::DBIC 'schema'; use Dancer::Plugin::Passphrase; +use App::Netdisco::Util::Statistics (); info "App::Netdisco $App::Netdisco::VERSION loaded."; @@ -139,7 +140,7 @@ sub deploy_db { say 'DB schema update complete.'; print color 'reset'; - system 'netdisco-do stats >/dev/null 2>&1'; + App::Netdisco::Util::Statistics::update_stats(); print color 'bold blue'; say 'Statistics updated.'; print color 'reset';