Squashed commit of the following:

commit c8f399b2d70248acde6bbabfa58ed4312bf1f8c2
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:53:14 2013 +0100

    version bump

commit 39d16adf38c142feeb1378c217a59783c5a12b44
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:43:37 2013 +0100

    initial discover triggers *walk

commit 29c334db714ac4846d024d815ffe1822889e879c
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 22:07:03 2013 +0100

    factor out body of poller and interactive worker role

commit db21f88e5a24ad99d808e70213cab1d83cc1ffee
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 20:58:07 2013 +0100

    fix logging when daemonized

commit 1f4a0539de7368273ddc7c24a8ffb0a663817e72
Author: Oliver Gorwits <oliver@cpan.org>
Date:   Tue Jun 11 17:49:44 2013 +0100

    limit discovery and remove duff constraint on device_ip

Netdisco/Changes

@@ -1,8 +1,20 @@
+2.008002 - 2013-06-11
+
+  [ENHANCEMENTS]
+
+  * Initial (bootstrap) discover now queues an arpwalk and macwalk as well
+
+  [BUG FIXES]
+
+  * Fix logging when daemonized
+  * Remove bad constraint on device_ip table
+  * Limit queueing of discovery to things which can be discovered
+
 2.008001 - 2013-06-11
 
   [BUG FIXES]
 
-  * more fixes for localenv discovery in scripts
+  * More fixes for localenv discovery in scripts
 
 2.008000 - 2013-06-09
 

Netdisco/MANIFEST

@@ -26,6 +26,7 @@ lib/App/Netdisco/Daemon/DB.pm
 lib/App/Netdisco/Daemon/DB/Result/Admin.pm
 lib/App/Netdisco/Daemon/Queue.pm
 lib/App/Netdisco/Daemon/Util.pm
+lib/App/Netdisco/Daemon/Worker/Common.pm
 lib/App/Netdisco/Daemon/Worker/Interactive.pm
 lib/App/Netdisco/Daemon/Worker/Interactive/DeviceActions.pm
 lib/App/Netdisco/Daemon/Worker/Interactive/PortActions.pm
@@ -93,6 +94,7 @@ lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-18-19-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-19-20-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-3-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-2-PostgreSQL.sql
+lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-3-4-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-4-5-PostgreSQL.sql
 lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-5-6-PostgreSQL.sql

Netdisco/META.yml

@@ -60,4 +60,4 @@ resources:
   homepage: http://netdisco.org/
   license: http://opensource.org/licenses/bsd-license.php
   repository: git://git.code.sf.net/p/netdisco/netdisco-ng
-version: 2.008001
+version: 2.008002

Netdisco/bin/netdisco-daemon-fg

@@ -69,8 +69,7 @@ sub build_tasks_list {
     user_begin => worker_factory('Interactive'),
   } if setting('workers')->{interactives};
 
-  info sprintf "MCE will load %s tasks: %s Manager, %s Scheduler, %s Poller, %s Interactive",
+  info sprintf "MCE will load: %s Manager, %s Scheduler, %s Poller, %s Interactive",
-    (scalar @$tasks),
     ((setting('workers')->{pollers} or setting('workers')->{interactives}) ? 1 : 0),
     (setting('housekeeping') ? 1 : 0),
     (setting('workers')->{pollers} || 0),

Netdisco/lib/App/Netdisco.pm

@@ -7,7 +7,7 @@ use 5.010_000;
 use File::ShareDir 'dist_dir';
 use Path::Class;
 
-our $VERSION = '2.008001';
+our $VERSION = '2.008002';
 
 BEGIN {
   if (not length ($ENV{DANCER_APPDIR} || '')

Netdisco/lib/App/Netdisco/Core/Discover.pm

@@ -3,7 +3,7 @@ package App::Netdisco::Core::Discover;
 use Dancer qw/:syntax :script/;
 use Dancer::Plugin::DBIC 'schema';
 
-use App::Netdisco::Util::Device 'get_device';
+use App::Netdisco::Util::Device qw/get_device is_discoverable/;
 use App::Netdisco::Util::DNS ':all';
 use NetAddr::IP::Lite ':lower';
 use Try::Tiny;
@@ -51,7 +51,7 @@ sub store_device {
   _set_canonical_ip($device, $snmp);
 
   my $hostname = hostname_from_ip($device->ip);
-  $device->dns($hostname) if length $hostname;
+  $device->dns($hostname) if $hostname;
   my $localnet = NetAddr::IP::Lite->new('127.0.0.0/8');
 
   # build device aliases suitable for DBIC
@@ -791,30 +791,26 @@ The Device database object can be a fresh L<DBIx::Class::Row> object which is
 not yet stored to the database.
 
 Any discovered neighbor unknown to Netdisco will have a C<discover> job
-immediately queued (subject to the filtering by the C<discover_no_type>
+immediately queued (subject to the filtering by the C<discover_*> settings).
-setting).
 
 =cut
 
 sub discover_new_neighbors {
   my @to_discover = store_neighbors(@_);
 
-  # only enqueue if device is not already discovered, and
+  # only enqueue if device is not already discovered,
-  # discover_no_type config permits the discovery
+  # discover_* config permits the discovery
   foreach my $neighbor (@to_discover) {
       my ($ip, $remote_type) = @$neighbor;
 
       my $device = get_device($ip);
       next if $device->in_storage;
 
-      if ($remote_type) {
+      if (not is_discoverable($device, $remote_type)) {
-          if (scalar grep {$remote_type =~ m/$_/}
+          debug sprintf
-                          @{setting('discover_no_type') || []}) {
+            ' queue - %s, type [%s] excluded by discover_* config',
-              debug sprintf
+            $ip, $remote_type;
-                ' queue - %s, type [%s] excluded by discover_no_type',
+          next;
-                $ip, $remote_type;
-              next;
-          }
       }
 
       # could fail if queued job already exists

Netdisco/lib/App/Netdisco/DB.pm

@@ -8,7 +8,7 @@ use base 'DBIx::Class::Schema';
 
 __PACKAGE__->load_namespaces;
 
-our $VERSION = 20; # schema version used for upgrades, keep as integer
+our $VERSION = 21; # schema version used for upgrades, keep as integer
 
 use Path::Class;
 use File::Basename;

Netdisco/lib/App/Netdisco/DB/schema_versions/App-Netdisco-DB-20-21-PostgreSQL.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE device_ip DROP CONSTRAINT "device_ip_alias";
+
+COMMIT;

Netdisco/lib/App/Netdisco/Daemon/Util.pm

@@ -4,10 +4,11 @@ package App::Netdisco::Daemon::Util;
 
 use base 'Exporter';
 our @EXPORT = ();
-our @EXPORT_OK = qw/ job_done job_error /;
+our @EXPORT_OK = qw/ job_done job_error job_defer /;
 our %EXPORT_TAGS = (all => \@EXPORT_OK);
 
 sub job_done  { return ('done',  shift) }
 sub job_error { return ('error', shift) }
+sub job_defer { return ('defer', shift) }
 
 1;

Netdisco/lib/App/Netdisco/Daemon/Worker/Common.pm

@@ -0,0 +1,86 @@
+package App::Netdisco::Daemon::Worker::Common;
+
+use Dancer qw/:moose :syntax :script/;
+use Dancer::Plugin::DBIC 'schema';
+use Try::Tiny;
+
+use Role::Tiny;
+use namespace::clean;
+
+requires qw/worker_type worker_name munge_action/;
+
+sub worker_body {
+  my $self = shift;
+  my $wid = $self->wid;
+
+  my $type = $self->worker_type;
+  my $name = $self->worker_name;
+
+  while (1) {
+      debug "$type ($wid): asking for a job";
+      my $jobs = $self->do('take_jobs', $self->wid, $name);
+
+      foreach my $candidate (@$jobs) {
+          # create a row object so we can use column accessors
+          # use the local db schema in case it is accidentally 'stored'
+          # (will throw an exception)
+          my $job = schema('daemon')->resultset('Admin')
+                      ->new_result($candidate);
+          my $jid = $job->job;
+
+          my $target = $self->munge_action($job->action);
+          next unless $self->can($target);
+          debug "$type ($wid): can ${target}() for job $jid";
+
+          # do job
+          my ($status, $log);
+          try {
+              $job->started(scalar localtime);
+              info sprintf "$type (%s): starting %s job(%s) at %s",
+                $wid, $target, $jid, $job->started;
+              ($status, $log) = $self->$target($job);
+          }
+          catch {
+              $status = 'error';
+              $log = "error running job: $_";
+              $self->sendto('stderr', $log ."\n");
+          };
+
+          $self->close_job($job, $status, $log);
+      }
+
+      debug "$type ($wid): sleeping now...";
+      sleep( setting('workers')->{sleep_time} || 5 );
+  }
+}
+
+sub close_job {
+  my ($self, $job, $status, $log) = @_;
+  my $type = $self->worker_type;
+  my $now = scalar localtime;
+
+  info sprintf "$type (%s): wrapping up %s job(%s) - status %s at %s",
+    $self->wid, $job->action, $job->job, $status, $now;
+
+  # lock db row and either defer or complete the job
+  try {
+      if ($status eq 'defer') {
+          schema('netdisco')->resultset('Admin')
+            ->find($job->job, {for => 'update'})
+            ->update({ status => 'queued' });
+      }
+      else {
+          schema('netdisco')->resultset('Admin')
+            ->find($job->job, {for => 'update'})
+            ->update({
+              status => $status,
+              log => $log,
+              started => $job->started,
+              finished => $now,
+            });
+      }
+  }
+  catch { $self->sendto('stderr', "error closing job: $_\n") };
+}
+
+1;

Netdisco/lib/App/Netdisco/Daemon/Worker/Interactive.pm

@@ -1,75 +1,17 @@
 package App::Netdisco::Daemon::Worker::Interactive;
 
-use Dancer qw/:moose :syntax :script/;
-use Dancer::Plugin::DBIC 'schema';
-use Try::Tiny;
-
 use Role::Tiny;
 use namespace::clean;
 
+# main worker body
+with 'App::Netdisco::Daemon::Worker::Common';
+
 # add dispatch methods for interactive actions
 with 'App::Netdisco::Daemon::Worker::Interactive::DeviceActions',
      'App::Netdisco::Daemon::Worker::Interactive::PortActions';
 
-sub worker_body {
+sub worker_type { 'int' }
-  my $self = shift;
+sub worker_name { 'Interactive' }
-  my $wid = $self->wid;
+sub munge_action { 'set_' . $_[1] }
-
-  while (1) {
-      debug "int ($wid): asking for a job";
-      my $jobs = $self->do('take_jobs', $self->wid, 'Interactive');
-
-      foreach my $candidate (@$jobs) {
-          # create a row object so we can use column accessors
-          # use the local db schema in case it is accidentally 'stored'
-          # (will throw an exception)
-          my $job = schema('daemon')->resultset('Admin')
-                      ->new_result($candidate);
-          my $jid = $job->job;
-
-          my $target = 'set_'. $job->action;
-          next unless $self->can($target);
-          debug "int ($wid): can ${target}() for job $jid";
-
-          # do job
-          my ($status, $log);
-          try {
-              $job->started(scalar localtime);
-              info sprintf "int (%s): starting %s job(%s) at %s",
-                $wid, $target, $jid, $job->started;
-              ($status, $log) = $self->$target($job);
-          }
-          catch {
-              $status = 'error';
-              $log = "error running job: $_";
-              $self->sendto('stderr', $log ."\n");
-          };
-
-          $self->close_job($job, $status, $log);
-      }
-
-      debug "int ($wid): sleeping now...";
-      sleep( setting('workers')->{sleep_time} || 5 );
-  }
-}
-
-sub close_job {
-  my ($self, $job, $status, $log) = @_;
-  my $now = scalar localtime;
-  info sprintf "int (%s): wrapping up set_%s job(%s) - status %s at %s",
-    $self->wid, $job->action, $job->job, $status, $now;
-
-  try {
-      schema('netdisco')->resultset('Admin')
-        ->find($job->job, {for => 'update'})
-        ->update({
-          status => $status,
-          log => $log,
-          started => $job->started,
-          finished => $now,
-        });
-  }
-  catch { $self->sendto('stderr', "error closing job: $_\n") };
-}
 
 1;

Netdisco/lib/App/Netdisco/Daemon/Worker/Manager.pm

@@ -95,11 +95,10 @@ sub lock_job {
   # lock db row and update to show job has been picked
   try {
       schema('netdisco')->txn_do(sub {
-          my $row = schema('netdisco')->resultset('Admin')->find(
+          schema('netdisco')->resultset('Admin')->find(
-            {job => $job->job, status => 'queued'}, {for => 'update'}
+            {job => $job->job, status => 'queued'},
-          );
+            {for => 'update'}
-
+          )->update({ status => "queued-$fqdn" });
-          $row->update({status => "queued-$fqdn"});
       });
       $happy = 1;
   };

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller.pm

@@ -1,77 +1,18 @@
 package App::Netdisco::Daemon::Worker::Poller;
 
-use Dancer qw/:moose :syntax :script/;
-use Dancer::Plugin::DBIC 'schema';
-
-use Try::Tiny;
-
 use Role::Tiny;
 use namespace::clean;
 
+# main worker body
+with 'App::Netdisco::Daemon::Worker::Common';
+
 # add dispatch methods for poller tasks
-with 'App::Netdisco::Daemon::Worker::Poller::Device';
+with 'App::Netdisco::Daemon::Worker::Poller::Device',
-with 'App::Netdisco::Daemon::Worker::Poller::Arpnip';
+     'App::Netdisco::Daemon::Worker::Poller::Arpnip',
-with 'App::Netdisco::Daemon::Worker::Poller::Macsuck';
+     'App::Netdisco::Daemon::Worker::Poller::Macsuck';
 
-sub worker_body {
+sub worker_type { 'pol' }
-  my $self = shift;
+sub worker_name { 'Poller' }
-  my $wid = $self->wid;
+sub munge_action { $_[1] }
-
-  while (1) {
-      debug "poll ($wid): asking for a job";
-      my $jobs = $self->do('take_jobs', $self->wid, 'Poller');
-
-      foreach my $candidate (@$jobs) {
-          # create a row object so we can use column accessors
-          # use the local db schema in case it is accidentally 'stored'
-          # (will throw an exception)
-          my $job = schema('daemon')->resultset('Admin')
-                      ->new_result($candidate);
-          my $jid = $job->job;
-          my $target = $job->action;
-
-          next unless $self->can($target);
-          debug "poll ($wid): can ${target}() for job $jid";
-
-          # do job
-          my ($status, $log);
-          try {
-              $job->started(scalar localtime);
-              info sprintf "poll (%s): starting %s job(%s) at %s",
-                $wid, $target, $jid, $job->started;
-              ($status, $log) = $self->$target($job);
-          }
-          catch {
-              $status = 'error';
-              $log = "error running job: $_";
-              $self->sendto('stderr', $log ."\n");
-          };
-
-          $self->close_job($job, $status, $log);
-      }
-
-      debug "poll ($wid): sleeping now...";
-      sleep( setting('workers')->{sleep_time} || 5 );
-  }
-}
-
-sub close_job {
-  my ($self, $job, $status, $log) = @_;
-  my $now = scalar localtime;
-  info sprintf "poll (%s): wrapping up %s job(%s) - status %s at %s",
-    $self->wid, $job->action, $job->job, $status, $now;
-
-  try {
-      schema('netdisco')->resultset('Admin')
-        ->find($job->job, {for => 'update'})
-        ->update({
-          status => $status,
-          log => $log,
-          started => $job->started,
-          finished => $now,
-        });
-  }
-  catch { $self->sendto('stderr', "error closing job: $_\n") };
-}
 
 1;

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Arpnip.pm

@@ -20,6 +20,16 @@ sub arpwalk {
   my $devices = schema('netdisco')->resultset('Device')->get_column('ip');
   my $jobqueue = schema('netdisco')->resultset('Admin');
 
+  if ($job->subaction and $job->subaction eq 'after-discoverall') {
+      # make sure there are no incomplete discover jobs queued
+      my $discover = $jobqueue->search(
+        { action => 'discover', status => { -like => 'queued%' } }
+      )->count;
+
+      return job_defer("Deferred arpwalk due to pending discover jobs")
+        if $discover;
+  }
+
   schema('netdisco')->txn_do(sub {
     # clean up user submitted jobs older than 1min,
     # assuming skew between schedulers' clocks is not greater than 1min

Netdisco/lib/App/Netdisco/Daemon/Worker/Poller/Device.pm

@@ -50,6 +50,10 @@ sub discover {
   my $host = NetAddr::IP::Lite->new($job->device);
   my $device = get_device($host->addr);
 
+  if ($device->ip eq '0.0.0.0') {
+      return job_error("discover failed: no device param (need -d ?)");
+  }
+
   if ($device->in_storage
       and $device->vendor and $device->vendor eq 'netdisco') {
       return job_done("Skipped discover for pseudo-device $host");

Netdisco/lib/App/Netdisco/Manual/Configuration.pod

@@ -65,13 +65,6 @@ Value: C<debug|warning|error>. Default: C<warning>.
 The log level used by Netdisco. It's useful to see warning messages from the
 backend poller, as this can highlight broken topology.
 
-=head3 C<logger>
-
-Value: C<console|file>. Default: C<file>.
-
-Destination for log messages. Console means standard ouput. When set to
-C<file>, the default destination is the C<${HOME}/logs> directory.
-
 =head3 C<logger_format>
 
 Value: Format String. Default: C<< '[%P] %L @%D> %m' >>.
@@ -411,6 +404,15 @@ Value: Boolean. Default: C<false>.
 
 Whether to show a stack trace when an error is caught in the web frontend.
 
+=head3 C<logger>
+
+Value: C<console|file>. Default: C<console>.
+
+Destination for log messages. Should usually be C<console>, which does the
+right thing when running foreground apps, and is also captured to
+C<${HOME}/logs> when running daemonized. Only change this if you know what
+you're doing.
+
 =head3 C<engines>
 
 Value: Settings Tree.

Netdisco/lib/App/Netdisco/Util/Device.pm

@@ -67,11 +67,16 @@ Returns false if the host is not permitted to discover the target device.
 =cut
 
 sub is_discoverable {
-  my $q = shift;
+  my ($ip, $remote_type) = @_;
+  my $device = get_device($ip) or return 0;
+
+  if ($remote_type) {
+      return 0 if
+        scalar grep {$remote_type =~ m/$_/}
+                    @{setting('discover_no_type') || []};
+  }
 
-  my $device = get_device($q) or return 0;
   my $addr = NetAddr::IP::Lite->new($device->ip);
-
   my $discover_no   = setting('discover_no') || [];
   my $discover_only = setting('discover_only') || [];
 

Netdisco/lib/App/Netdisco/Util/SNMP.pm

@@ -154,7 +154,7 @@ sub _build_mibdirs {
 
 sub _get_mibdirs_content {
   my $home = shift;
-  warning 'Netdisco SNMP work will be really slow - loading ALL MIBs. Please set mibdirs.';
+  warning 'Netdisco SNMP work will be slow - loading ALL MIBs. Consider setting mibdirs.';
   my @list = map {s|$home/||; $_} grep {-d} glob("$home/*");
   return \@list;
 }

Netdisco/lib/App/Netdisco/Web/AdminTask.pm

@@ -16,7 +16,7 @@ sub add_job {
     }
 
     try {
-    # job might already be in the queue, so this could die
+    # jobs might already be in the queue, so this could die
         schema('netdisco')->resultset('Admin')->create({
           ($device ? (device => $device->addr) : ()),
           action => $jobtype,
@@ -24,6 +24,23 @@ sub add_job {
           username => session('user'),
           userip => request->remote_address,
         });
+
+        if (param('extra') and param('extra') eq 'with-walk') {
+            schema('netdisco')->resultset('Admin')->create({
+              action => 'macwalk',
+              subaction => 'after-discoverall',
+              status => 'queued',
+              username => session('user'),
+              userip => request->remote_address,
+            });
+            schema('netdisco')->resultset('Admin')->create({
+              action => 'arpwalk',
+              subaction => 'after-discoverall',
+              status => 'queued',
+              username => session('user'),
+              userip => request->remote_address,
+            });
+        }
     };
 }
 

Netdisco/share/config.yml

@@ -10,7 +10,7 @@
 # ----------------
 
 log: 'warning'
-logger: 'file'
+logger: 'console'
 logger_format: '[%P] %L @%D> %m'
 
 # ------------

Netdisco/share/views/index.tt

@@ -52,6 +52,7 @@
         <form method="post" action="[% uri_for('/admin/discover') %]">
           <div class="form-horizontal">
             <input placeholder="Device hostname or IP" class="span4" name="device" type="text"/>
+            <input type="hidden" name="extra" value="with-walk"/>
             <button type="submit" class="btn btn-info">Discover</button>
           </div>
         </form>