From 5ff7d6fe471df1f6804dc7fc7c6a288320fc75f7 Mon Sep 17 00:00:00 2001 From: Oliver Gorwits Date: Sun, 19 Nov 2017 13:34:35 +0000 Subject: [PATCH] Merge the backend worker plugins branch og-coreplugins Squashed commit of the following: commit 86d0f61d0b75fe6f2945c7fb3ca3483b12c9822e Author: Oliver Gorwits Date: Thu Nov 16 22:26:32 2017 +0000 fix typo commit 5aff19621cc8bbe4bc0fb0170911e01cf68fa273 Author: Oliver Gorwits Date: Thu Nov 16 22:10:18 2017 +0000 fix use of snmp_connect_ip which does not work for SNMPv3 commit 68a56d35bb106b7817dfc6e7581246141826e5f3 Author: Oliver Gorwits Date: Thu Nov 16 20:50:16 2017 +0000 no need for Array::Iterator even though it was cute commit 71ee869c025f3e1f95d7895e44f8c0aca1eacec0 Author: Oliver Gorwits Date: Wed Nov 15 22:14:47 2017 +0000 additional doc examples commit 620b3fe544fba56591a6dfef1d04d1825893fd50 Author: Oliver Gorwits Date: Wed Nov 15 22:09:05 2017 +0000 stash workers within poller instance, and load plugins explicitly commit 243136558310cf8fc3fcc05d30691ef8a50b1b35 Author: Oliver Gorwits Date: Mon Nov 13 22:17:11 2017 +0000 better fix for duplicate module entity index commit a400b267042c3f4bfed93d8239014ed9d7b02b7e Author: Oliver Gorwits Date: Mon Nov 13 22:14:42 2017 +0000 add ignore interfaces for HPE routers commit 1502ec196657229908f3af3cd573ca85858c7796 Author: Oliver Gorwits Date: Mon Nov 13 22:08:02 2017 +0000 bug fixes after testing on a real network commit 840b6b406960e70dec3798703d9a7b89e4b9dfba Author: Oliver Gorwits Date: Sun Nov 12 20:38:35 2017 +0000 add tests commit 2de36c69ba47a3b0ebbd06abeeab02c97ffbb3b9 Author: Oliver Gorwits Date: Sun Nov 12 00:14:21 2017 +0000 some reengineering to support proper testing commit c5f138fe624cfcb8de3d7e3f0fa0dbbb60c34874 Author: Oliver Gorwits Date: Sat Nov 11 14:43:53 2017 +0000 correct algorithm on finalise status, correct logging commit 98442a23084c97caf38c346cf880dec1794337d4 Author: Oliver Gorwits Date: Thu Nov 9 21:49:45 2017 +0000 bug fixes commit e0c6615c87b3a4d4518adf6baca6c2aed5cf886d Author: Oliver Gorwits Date: Wed Nov 8 20:29:33 2017 +0000 fix bugs commit 1eeaba441d2dc6a3b98cba548727df3fee4eddb8 Author: Oliver Gorwits Date: Tue Nov 7 22:30:55 2017 +0000 finish refactor to new desired behaviour (buggy?) commit 7edfe88f256c083b608af80e2c99f0deb3925709 Author: Oliver Gorwits Date: Mon Nov 6 22:50:51 2017 +0000 fix to work, and correct namespace check commit 25907d3544b86bb357d1cc7b29d47bf133b32443 Author: Oliver Gorwits Date: Mon Nov 6 21:26:01 2017 +0000 move status tracking and checking inside job instance commit 4436150bf4b37583f4f3fbeec388a6703e17454d Author: Oliver Gorwits Date: Sun Nov 5 20:54:28 2017 +0000 remove global rubbish commit 28b016e7136baad49ba3204cae6b37c8f8b11fab Author: Oliver Gorwits Date: Sat Nov 4 23:31:51 2017 +0000 fix docs commit 650f6c719b84e79f5db7f338d7cbeeb5c5b44124 Author: Oliver Gorwits Date: Sat Nov 4 23:22:12 2017 +0000 tidy line commit 10f78d5dbe4702c13fc200cb0146b7d75b4c5c60 Author: Oliver Gorwits Date: Sat Nov 4 23:06:20 2017 +0000 add priority and namespace to support fancy worker overrides commit b9f9816d0984c48bdead574c8350db8a51461b74 Author: Oliver Gorwits Date: Wed Oct 11 18:33:46 2017 +0100 release 2.036012_001 commit c33bf204a4a8946879e5e4618a4c974a6f064533 Merge: 5b7ce3f7 d3d81eb6 Author: Oliver Gorwits Date: Wed Oct 11 18:30:23 2017 +0100 Merge branch 'master' into og-coreplugins commit 5b7ce3f7970ee78c6589bc47cfe9e3281eef776b Author: Oliver Gorwits Date: Mon Oct 9 15:46:09 2017 +0100 cannot Sereal::Encode DBIC row commit 0a575f02ba7bbe9632da51f13430b8ff75d92bd1 Author: Oliver Gorwits Date: Mon Oct 9 14:07:56 2017 +0100 fix bug in job->device init commit 207476950ddc0266bf5a41dbf17aacd122d67acc Author: Oliver Gorwits Date: Mon Oct 9 14:03:37 2017 +0100 default causes no attr to be created?! commit 912f2fa91fee87fa5029c826f4d40bcc434f127f Author: Oliver Gorwits Date: Sun Oct 8 18:43:51 2017 +0100 better debug logging commit dfeb9d9ddc114c0ad6f1779f60fbba285c1d0503 Author: Oliver Gorwits Date: Sun Oct 8 18:40:02 2017 +0100 make device_auth have driver setting for snmp entries commit 460c0c0ee9259fe3feb4713c0be164d5d6804a59 Merge: 3ccd107b 98423445 Author: Oliver Gorwits Date: Sun Oct 8 18:08:58 2017 +0100 Merge branch 'master' into og-coreplugins commit 3ccd107bd48357892263353ba93e37a171317522 Author: Oliver Gorwits Date: Sat Oct 7 14:13:58 2017 +0100 fix bug in device->has_layer commit a4b9bf203609bd765af9c5f3c07851b123aefdde Author: Oliver Gorwits Date: Sat Oct 7 13:58:52 2017 +0100 netdisco-do show takes a param for method in -p commit 4389cd0459d62b3dc7e9ee30d9dbf91ff26a7e77 Author: Oliver Gorwits Date: Sat Oct 7 13:36:06 2017 +0100 fix to only check last poll on devices in storage commit 58d0fbddda85f01c2607e708725bc364740602d7 Author: Oliver Gorwits Date: Sat Oct 7 13:21:13 2017 +0100 do not run discover parts if properties failed to complete commit b52aaaf1a1cc58a39212a1fe7a2d8d9e7f429803 Author: Oliver Gorwits Date: Sat Oct 7 13:08:46 2017 +0100 fix typo commit 41be9269215421a3d8a7a0e8c661279e999deac7 Author: Oliver Gorwits Date: Sat Oct 7 13:04:45 2017 +0100 run all check workers commit a41d1149651163ad00fdf6aaa66d4ea12ab3859d Author: Oliver Gorwits Date: Sat Oct 7 13:02:46 2017 +0100 fix driver config commit b10908a1387815176c1d59f961beb2bfc0cb6a7b Author: Oliver Gorwits Date: Sat Oct 7 12:43:50 2017 +0100 use vars() cache between phases commit 08b34e083db3a7c872ddced728c619796e3dc5e1 Author: Oliver Gorwits Date: Sat Oct 7 11:39:17 2017 +0100 remove die() calls commit b8108986fbdf3d13e47ed0a0b5f6d18a5512688d Author: Oliver Gorwits Date: Sat Oct 7 11:31:59 2017 +0100 phase fixups commit 273cbbc11b1d1cd1291b775aa3ea0575b519035a Author: Oliver Gorwits Date: Sat Oct 7 09:42:41 2017 +0100 change stage to phase commit 256c10bae5837d37f7c4990a7c1f60e5ce4706f7 Author: Oliver Gorwits Date: Sat Oct 7 09:35:14 2017 +0100 multi worker actions need not return done from all workers commit ee38bae48aa66d6339534edd204b7de4e8033832 Author: Oliver Gorwits Date: Sat Oct 7 09:05:25 2017 +0100 store result of worker if best for this phase so far commit 5bddfc73bad0a456a02d468ac24f5453fe5abb30 Author: Oliver Gorwits Date: Sat Oct 7 08:50:31 2017 +0100 auto debug-log worker return messages commit 8b660a89c042be5c492ac27964db3360cd12fd9d Author: Oliver Gorwits Date: Fri Oct 6 07:48:58 2017 +0100 bug fixes commit b58a5816a9b55e450899b1723f29941042d8fae4 Author: Oliver Gorwits Date: Fri Oct 6 07:44:20 2017 +0100 remove unnecessary check phases commit e44f06364a1de24ad16f91ba5c44e0c1e94f9d19 Author: Oliver Gorwits Date: Fri Oct 6 07:18:03 2017 +0100 fix unknown command check in netdisco-do commit 3af13f0dfe69f8a39da5e0ff6b819c2cc4695be5 Author: Oliver Gorwits Date: Fri Oct 6 07:15:59 2017 +0100 introduce noop and refactor checks in all workers commit 98463c8cad74c0498b46f0b68795079b072cd003 Author: Oliver Gorwits Date: Sun Oct 1 10:49:12 2017 +0100 no need to debug log if there are no hooks in phase commit 3b32e8431219fd4f889e5d035069d024e64cfbf7 Author: Oliver Gorwits Date: Sun Oct 1 08:18:13 2017 +0100 fiddle about with runner logic to fix exit states commit 8fdba38ee0b119fb4404501cde56e0657dd1869f Author: Oliver Gorwits Date: Fri Sep 29 08:01:42 2017 +0100 cannot reuse a worker as the job will be already set and the wrong plugins loaded commit a155d9cb77fe50f8be425fb0f6f2c18d58b3431f Author: Oliver Gorwits Date: Fri Sep 29 08:01:06 2017 +0100 should defer when we cannot connect to device commit 10b5f6cbc4c19f77ea54602b4f3c210e9aa5445e Author: Oliver Gorwits Date: Fri Sep 29 08:00:32 2017 +0100 fix bug in where workerconf acls are checked commit 2a74e0befa7fab51da11655ff96f0e3c620b4693 Author: Oliver Gorwits Date: Fri Sep 29 07:38:05 2017 +0100 can pass device instance to check_* commit 4256b117dfb79eef8f738de661a8656d1de70b5f Author: Oliver Gorwits Date: Fri Sep 29 07:27:14 2017 +0100 move device_auth build to be with community defaults setting commit a2de2c16165766441cdbcc22c63221852d7e9e49 Merge: 32be11c3 8dc4b9bc Author: Oliver Gorwits Date: Fri Sep 29 07:21:03 2017 +0100 Merge branch 'master' into og-coreplugins commit 32be11c3ffe90d3045f81b56d9f7ebf90763da0a Author: Oliver Gorwits Date: Thu Sep 21 00:09:29 2017 +0100 move remaining interactive actions to be plugins commit 3e41c93f5abfd954e504e709deb60a370a056270 Author: Oliver Gorwits Date: Wed Sep 20 21:47:50 2017 +0100 clean snmp handling commit 30a2d5dd863fe50393be6722dadce74e33283eff Author: Oliver Gorwits Date: Wed Sep 20 21:00:29 2017 +0100 make sure check plugins are loaded/run before phases commit 3454d95a84ba162d21cae76635d656e7e8c4c7a5 Author: Oliver Gorwits Date: Wed Sep 20 20:53:52 2017 +0100 capture result on main phase as well commit 559fa4f93fe5bea654433e62fa78376a57e75032 Author: Oliver Gorwits Date: Mon Sep 18 22:46:35 2017 +0100 build device_auth from communities commit 196929171957a3f3f3c9d4ef1bc0050992441787 Author: Oliver Gorwits Date: Mon Sep 18 22:04:22 2017 +0100 simplify to remove phases and fewer hooks commit 6f78032e28cfdcc37044ee1ec1dcaf41d3ef70e5 Author: Oliver Gorwits Date: Thu Sep 14 21:30:03 2017 +0100 add phase to test worker commit 6edd2dc879f00d9bf3b12de2d4863f79e73d97e0 Author: Oliver Gorwits Date: Wed Sep 13 21:51:40 2017 +0100 no need to list all plugins commit dfaeb34d8cc95c3522d2ba46aaee0cc33693dbd4 Author: Oliver Gorwits Date: Wed Sep 13 20:42:41 2017 +0100 add reset after messing with snmp context or community index commit 09214dce921884358f77f7a651e32656eb059a31 Author: Oliver Gorwits Date: Wed Sep 13 20:29:21 2017 +0100 no need to pass $snmp around commit 58cd488ccc2ec22c30ae5e646a6220864f807f83 Author: Oliver Gorwits Date: Wed Sep 13 19:22:40 2017 +0100 refactor layer and pseudo checks commit 753acc607ff3b793236f4c97b8a103f58a484a8a Author: Oliver Gorwits Date: Wed Sep 13 10:53:12 2017 +0100 use overloaded $device commit d5d39289d6962d03a324f53a84947594a7e9fb84 Author: Oliver Gorwits Date: Wed Sep 13 10:44:31 2017 +0100 rename init stage to check commit 1fdb0861834fb6a77e165701f368bf7bb7e2f86f Author: Oliver Gorwits Date: Tue Sep 12 08:12:12 2017 +0100 refactor to remove second loop commit 64a9491115e782d99fa8973731e24f6bcf8474f9 Author: Oliver Gorwits Date: Sun Sep 10 16:09:45 2017 +0100 change to init, first, second stages commit 5f2da69697a1b18735b608a5bbd14e23c2c473a7 Author: Oliver Gorwits Date: Sat Sep 9 22:26:04 2017 +0100 move discover and discoverall to worker plugins commit c6ebb7cf0717cd4f1861ed997846dcfaeebd42b7 Author: Oliver Gorwits Date: Sat Sep 9 16:44:32 2017 +0100 move arpnip and arpwalk to worker plugins commit 16a79463cb2a938e1054cbc83c897bdc70e47cee Author: Oliver Gorwits Date: Sat Sep 9 16:27:58 2017 +0100 set snmp driver on macsuck phase workers commit 9167e02de52f5d62a72922b4ba2ad7080719ce68 Author: Oliver Gorwits Date: Sat Sep 9 15:55:53 2017 +0100 move macsuck and macwalk to worker plugins (macsuck needs snmp scope guard) commit 68ca85643b152246b83ef04ceb22faed9a42dc94 Author: Oliver Gorwits Date: Sat Sep 9 14:56:15 2017 +0100 move expire and expirenodes to worker plugins commit 271ef1a25cfa223f28726c78d4e0524da865d8d5 Author: Oliver Gorwits Date: Sat Sep 9 14:46:00 2017 +0100 move nbtstat and nbtwalk to worker plugins commit e7508a9eca6b21d6c87817b6c53d47c1f82ebfab Author: Oliver Gorwits Date: Wed Sep 6 21:23:54 2017 +0100 move all netdisco-do action to worker plugins commit 707fc82b99eec52121b318246c0b21af59162d93 Author: Oliver Gorwits Date: Wed Sep 6 21:01:37 2017 +0100 remove psql code from netdisco-do and fix detection of misspelled action commit 411918e3f8506e9fd0b7d08c45e438de631e9c32 Author: Oliver Gorwits Date: Wed Sep 6 20:56:26 2017 +0100 only load worker plugins for the action commit 1f9740c0e20858896fa47aa038de1d6403ec1875 Author: Oliver Gorwits Date: Wed Sep 6 18:30:43 2017 +0100 shorten hook names commit a59c23de798c7d2fc3e246c3b00a98c68ff6d325 Author: Oliver Gorwits Date: Wed Sep 6 18:27:34 2017 +0100 make psql worker primary, add hook debug log commit 36c70220a2b10b7f9e3991eda604117bcfa1035a Author: Oliver Gorwits Date: Tue Sep 5 22:39:22 2017 +0100 allow two forms of worker declaration, and update docs commit a79cb9a9e45cc8ea7f3314b04b5cd83410a4cd68 Author: Oliver Gorwits Date: Tue Sep 5 22:10:53 2017 +0100 all the bug fixes and a working plugin!!!!!!!!! :-D commit 04896202e04e2f61200f03775bae7bce3cfa6f30 Author: Oliver Gorwits Date: Tue Sep 5 21:39:41 2017 +0100 refine runner commit 547fce2f3cead02111653de7a080cbf2eb3bb915 Author: Oliver Gorwits Date: Tue Sep 5 20:56:21 2017 +0100 hack the status class to regen if needed commit cd71a0b7a80bd5b7b27f0a95e4ebc1b51277c0b6 Author: Oliver Gorwits Date: Tue Sep 5 20:41:05 2017 +0100 move status update to job class commit c8e5cea4ed6f94b86b4bcf28e801d899ba517991 Author: Oliver Gorwits Date: Tue Sep 5 20:37:13 2017 +0100 objectify the running commit f48004fffa8bad38f23ab995ae1fd3c30fda7296 Author: Oliver Gorwits Date: Tue Sep 5 19:58:28 2017 +0100 bug squish commit 46ece568f6102a15be2076cb511699d13dad79d3 Author: Oliver Gorwits Date: Tue Sep 5 19:54:57 2017 +0100 implement runner?! commit fc9c60f707f33a0169e25513d626a7be6f197e0d Author: Oliver Gorwits Date: Tue Sep 5 19:28:38 2017 +0100 rename ok to is_ok and change slot names to avoid conflict with creators commit 3ee85383abced85e67793ea38c12e029965d3428 Author: Oliver Gorwits Date: Tue Sep 5 19:25:41 2017 +0100 skip worker when action is per-device but no creds commit 75abdad812e3b5302780070c6c0c2fc9448f3b10 Author: Oliver Gorwits Date: Mon Sep 4 21:54:37 2017 +0100 further work on retval handling from workers commit 4c1fdf4f928642fb10f1b2e7848403e10ea17e8c Author: Oliver Gorwits Date: Mon Sep 4 20:37:53 2017 +0100 move worker plugin loader to Worker.pm commit be0c5181a3c3779bf9d9f0a36e417f1a3fc6f2de Author: Oliver Gorwits Date: Mon Sep 4 20:35:42 2017 +0100 move Runner to Worker namespace commit 1c2cf924bc6e39f717433a3972ab7be609fe9c60 Author: Oliver Gorwits Date: Mon Sep 4 20:33:20 2017 +0100 worker roles in Role namespace commit 3099eda393f711b602baed424708215afa4e1295 Author: Oliver Gorwits Date: Mon Sep 4 20:30:58 2017 +0100 load workers when runner role is loaded commit a8c58a7b05da86caf3c913d55c99936e174bfc5a Author: Oliver Gorwits Date: Sun Sep 3 22:30:28 2017 +0100 initial broken implementation of the runner commit 49b5274c33ed610315ce1bfcf42015402231d950 Author: Oliver Gorwits Date: Sun Sep 3 19:04:20 2017 +0100 use run() mixin to exec action commit e0a666668a80012d1905905244de2ebb5e01f96f Author: Oliver Gorwits Date: Sun Sep 3 18:54:44 2017 +0100 fix pod; set status defaults; stub runner mixin commit 8eaa33770c782e17472a288e33d656176c31d5f2 Author: Oliver Gorwits Date: Sun Sep 3 18:45:00 2017 +0100 rename Core to Worker and move other packages around commit 4def0af0b02e4d2d1776c1374b6d6f2349716582 Author: Oliver Gorwits Date: Sun Sep 3 17:58:03 2017 +0100 better use of new status class commit 8675bf62c6b6c911bf01dd3735a46e2daa8a6034 Author: Oliver Gorwits Date: Sun Sep 3 17:27:38 2017 +0100 fix hook naming and implement primary workers commit ef1bb81f2ba8e4821afff052ee71c37c743d00a9 Author: Oliver Gorwits Date: Sun Sep 3 17:26:27 2017 +0100 new backend status class commit 5f50dfadf1e376b8a6eec352d4bde8402cc847dc Author: Oliver Gorwits Date: Sun Sep 3 16:51:55 2017 +0100 new Backend package to load core plugins commit 3baa7a818af89449baadc6e25003cc6acde84551 Author: Oliver Gorwits Date: Sun Sep 3 16:22:29 2017 +0100 remove unnecessary Worker::Common role commit 36b4adcc0655e8b571cbb106c9848327403caf28 Author: Oliver Gorwits Date: Sun Sep 3 16:17:29 2017 +0100 disambiguate util/backend package and remove backend prelaod commit 98bff731bd524b39115e60ebc75895c8350caff2 Author: Oliver Gorwits Date: Sat Sep 2 08:25:06 2017 +0100 settle on a design for hook override, I think commit fe5c16a16dc1b8ec15fed170669219e2c6c4501e Author: Oliver Gorwits Date: Wed Aug 30 20:37:36 2017 +0100 rework docs to be more clear and reflect new operation commit b34ba1977c7b1576e7eebf1da390052fc4fb58a0 Merge: 31d1977f c34ed61d Author: Oliver Gorwits Date: Mon Aug 21 21:17:46 2017 +0100 Merge branch 'master' into og-coreplugins commit 31d1977f1e4b4465dbb75be3d9a6c0fbfadf189f Author: Oliver Gorwits Date: Mon Aug 14 18:11:42 2017 +0100 Revert "move expire code to be initial plugin pilot (broken)" I think we'll only do the new backend code for jobs with a device. This reverts commit 07998b72d961f70a9843f172adfebc50a2d329da. commit 61dc80aff8af8e635c77e4672c766acb31a8cce4 Merge: 07998b72 ade02db1 Author: Oliver Gorwits Date: Mon Aug 14 18:10:29 2017 +0100 Merge branch 'master' into og-coreplugins commit 07998b72d961f70a9843f172adfebc50a2d329da Author: Oliver Gorwits Date: Sat Aug 5 22:15:00 2017 +0100 move expire code to be initial plugin pilot (broken) commit 685ec021080269e8f5dbfe4b940c8037b8289ec3 Author: Oliver Gorwits Date: Sat Aug 5 22:10:58 2017 +0100 pass $job to the core worker commit d6523fe5432e3a7bc91b5e0dc57e3e57a451d97a Author: Oliver Gorwits Date: Sat Aug 5 22:01:49 2017 +0100 $job->device is always a DBIC row commit ee6deea01b807134d6388feb96a1044675c98548 Author: Oliver Gorwits Date: Sat Aug 5 18:12:34 2017 +0100 load plugins commit fd80096ca2e94fd2a47ef5dd2e4cf00254345a63 Author: Oliver Gorwits Date: Sat Aug 5 16:53:16 2017 +0100 rename all the things commit 464c42d1f537a2189797e883e4d20d3a07883e48 Author: Oliver Gorwits Date: Wed Aug 2 10:19:16 2017 +0100 use Scope::Guard to reduce device_auth commit ec041dafd266df116e4b0f716121cf7031593152 Author: Oliver Gorwits Date: Tue Aug 1 15:34:37 2017 +0100 the other way around commit 33d2fe13bdf07c7a09c0a23ae45e3a2e11164364 Author: Oliver Gorwits Date: Mon Jul 31 17:57:29 2017 +0100 fix pod commit 3faee1cf16e9f3e81aa674ff9b96d436d4f52d57 Author: Oliver Gorwits Date: Mon Jul 31 17:55:10 2017 +0100 remove need for instance() call commit c6d0f1c035eb8ce555097bc481fc62ef3b15c20c Author: Oliver Gorwits Date: Wed Jul 26 13:51:23 2017 +0100 add doc note on accessing transports commit dca4b4fc030f851d55eb5ec52efbc62e593a3eed Author: Oliver Gorwits Date: Wed Jul 26 11:50:10 2017 +0100 add backend driver documentation commit 052a2acd79c96fc89c612ba2332cf05631517348 Author: Oliver Gorwits Date: Wed Jul 26 10:16:58 2017 +0100 rename web plugins doc commit 69c9a6393ac1a0d3e95cd8e349e7fdd16bdfa113 Author: Oliver Gorwits Date: Wed Jul 26 10:12:42 2017 +0100 rename args to driverconf commit 2586a36f8c31bf29c099fbeffbef74a8e7775715 Author: Oliver Gorwits Date: Tue Jul 25 22:41:10 2017 +0100 new version of core plugin manager with better config and filters commit 4056831f994a2d5131f9c19c5eff7391f0cefd8a Author: Oliver Gorwits Date: Tue Jul 25 20:53:56 2017 +0100 change SNMP to be a cached transport singleton commit c31030ef7016095dc80446fcda267e2a4590abeb Author: Oliver Gorwits Date: Sun Jul 23 13:46:27 2017 +0100 fixes because Dancer docs are a mess! commit f65ef90b861db3d736f59f2d01921135e7e555d5 Author: Oliver Gorwits Date: Sat Jul 22 08:11:36 2017 +0100 rename snmp_auth to device_auth and include a little doc on transports commit d61556e1cf63d5419bc1e3db9712e191019e0bac Author: Oliver Gorwits Date: Sat Jul 22 07:54:26 2017 +0100 plugin config added commit de8de563084a52222884ff5396b1de491fc329e2 Author: Oliver Gorwits Date: Wed Jul 12 21:38:31 2017 +0100 initial core plugin implementation --- Build.PL | 7 +- Changes | 2 + MANIFEST | 70 +- META.json | 204 +++- META.yml | 143 ++- bin/netdisco-backend-fg | 20 +- bin/netdisco-do | 164 +-- lib/App/Netdisco.pm | 2 +- lib/App/Netdisco/Backend/Job.pm | 146 ++- .../Backend/{Worker => Role}/Manager.pm | 14 +- .../{Worker/Common.pm => Role/Poller.pm} | 22 +- .../Backend/{Worker => Role}/Scheduler.pm | 8 +- lib/App/Netdisco/Backend/Util.pm | 17 - .../Worker/Interactive/DeviceActions.pm | 50 - .../Backend/Worker/Interactive/PortActions.pm | 159 --- lib/App/Netdisco/Backend/Worker/Poller.pm | 18 - .../Netdisco/Backend/Worker/Poller/Arpnip.pm | 18 - .../Netdisco/Backend/Worker/Poller/Common.pm | 99 -- .../Netdisco/Backend/Worker/Poller/Device.pm | 100 -- .../Netdisco/Backend/Worker/Poller/Macsuck.pm | 18 - .../Netdisco/Backend/Worker/Poller/Nbtstat.pm | 73 -- lib/App/Netdisco/Configuration.pm | 4 + lib/App/Netdisco/Core/Discover.pm | 996 ------------------ lib/App/Netdisco/DB/Result/Device.pm | 25 + lib/App/Netdisco/JobQueue/PostgreSQL.pm | 8 +- lib/App/Netdisco/Manual/Configuration.pod | 72 +- lib/App/Netdisco/Manual/Developing.pod | 2 +- ...itingPlugins.pod => WritingWebPlugins.pod} | 4 +- lib/App/Netdisco/Manual/WritingWorkers.pod | 226 ++++ lib/App/Netdisco/Transport/SNMP.pm | 312 ++++++ lib/App/Netdisco/Util/Device.pm | 21 +- lib/App/Netdisco/Util/{Backend.pm => MCE.pm} | 2 +- lib/App/Netdisco/{Core => Util}/Nbtstat.pm | 17 +- lib/App/Netdisco/Util/SNMP.pm | 425 ++------ lib/App/Netdisco/Web.pm | 2 +- lib/App/Netdisco/Web/Plugin.pm | 3 +- lib/App/Netdisco/Worker/Loader.pm | 55 + lib/App/Netdisco/Worker/Plugin.pm | 164 +++ lib/App/Netdisco/Worker/Plugin/Arpnip.pm | 31 + .../Plugin/Arpnip/Nodes.pm} | 110 +- .../Netdisco/Worker/Plugin/Arpnip/Subnets.pm | 74 ++ lib/App/Netdisco/Worker/Plugin/Arpwalk.pm | 31 + lib/App/Netdisco/Worker/Plugin/Contact.pm | 43 + lib/App/Netdisco/Worker/Plugin/Delete.pm | 24 + lib/App/Netdisco/Worker/Plugin/Discover.pm | 28 + .../Worker/Plugin/Discover/CanonicalIP.pm | 80 ++ .../Worker/Plugin/Discover/Entities.pm | 95 ++ .../Worker/Plugin/Discover/Neighbors.pm | 338 ++++++ .../Worker/Plugin/Discover/PortPower.pm | 81 ++ .../Worker/Plugin/Discover/Properties.pm | 245 +++++ .../Netdisco/Worker/Plugin/Discover/VLANs.pm | 95 ++ .../Worker/Plugin/Discover/Wireless.pm | 85 ++ .../Worker/Plugin/Discover/WithNodes.pm | 38 + lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm | 31 + .../Expiry.pm => Worker/Plugin/Expire.pm} | 41 +- lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm | 30 + lib/App/Netdisco/Worker/Plugin/Graph.pm | 14 + lib/App/Netdisco/Worker/Plugin/Location.pm | 43 + lib/App/Netdisco/Worker/Plugin/Macsuck.pm | 31 + .../Plugin/Macsuck/Nodes.pm} | 203 +--- .../Worker/Plugin/Macsuck/WirelessNodes.pm | 82 ++ lib/App/Netdisco/Worker/Plugin/Macwalk.pm | 31 + lib/App/Netdisco/Worker/Plugin/Monitor.pm | 14 + lib/App/Netdisco/Worker/Plugin/Nbtstat.pm | 21 + .../Netdisco/Worker/Plugin/Nbtstat/Core.pm | 50 + lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm | 31 + lib/App/Netdisco/Worker/Plugin/PortControl.pm | 76 ++ lib/App/Netdisco/Worker/Plugin/PortName.pm | 54 + lib/App/Netdisco/Worker/Plugin/Power.pm | 69 ++ lib/App/Netdisco/Worker/Plugin/Psql.pm | 39 + lib/App/Netdisco/Worker/Plugin/Renumber.pm | 39 + lib/App/Netdisco/Worker/Plugin/Show.pm | 37 + lib/App/Netdisco/Worker/Plugin/Stats.pm | 14 + lib/App/Netdisco/Worker/Plugin/Vlan.pm | 30 + lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm | 61 ++ lib/App/Netdisco/Worker/Runner.pm | 85 ++ lib/App/Netdisco/Worker/Status.pm | 105 ++ share/config.yml | 54 + share/environments/deployment.yml | 2 +- xt/30-backend-workers.t | 118 +++ .../App/NetdiscoX/Worker/Plugin/TestFive.pm | 17 + .../App/NetdiscoX/Worker/Plugin/TestFour.pm | 17 + xt/lib/App/NetdiscoX/Worker/Plugin/TestOne.pm | 17 + .../App/NetdiscoX/Worker/Plugin/TestThree.pm | 17 + xt/lib/App/NetdiscoX/Worker/Plugin/TestTwo.pm | 17 + 85 files changed, 4078 insertions(+), 2502 deletions(-) rename lib/App/Netdisco/Backend/{Worker => Role}/Manager.pm (94%) rename lib/App/Netdisco/Backend/{Worker/Common.pm => Role/Poller.pm} (76%) rename lib/App/Netdisco/Backend/{Worker => Role}/Scheduler.pm (96%) delete mode 100644 lib/App/Netdisco/Backend/Util.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Common.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Device.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm delete mode 100644 lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm delete mode 100644 lib/App/Netdisco/Core/Discover.pm rename lib/App/Netdisco/Manual/{WritingPlugins.pod => WritingWebPlugins.pod} (99%) create mode 100644 lib/App/Netdisco/Manual/WritingWorkers.pod create mode 100644 lib/App/Netdisco/Transport/SNMP.pm rename lib/App/Netdisco/Util/{Backend.pm => MCE.pm} (91%) rename lib/App/Netdisco/{Core => Util}/Nbtstat.pm (91%) create mode 100644 lib/App/Netdisco/Worker/Loader.pm create mode 100644 lib/App/Netdisco/Worker/Plugin.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpnip.pm rename lib/App/Netdisco/{Core/Arpnip.pm => Worker/Plugin/Arpnip/Nodes.pm} (52%) create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Arpwalk.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Contact.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Delete.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm rename lib/App/Netdisco/{Backend/Worker/Poller/Expiry.pm => Worker/Plugin/Expire.pm} (65%) create mode 100644 lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Graph.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Location.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Macsuck.pm rename lib/App/Netdisco/{Core/Macsuck.pm => Worker/Plugin/Macsuck/Nodes.pm} (70%) create mode 100644 lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Macwalk.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Monitor.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Nbtstat.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/PortControl.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/PortName.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Power.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Psql.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Renumber.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Show.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Stats.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Vlan.pm create mode 100644 lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm create mode 100644 lib/App/Netdisco/Worker/Runner.pm create mode 100644 lib/App/Netdisco/Worker/Status.pm create mode 100644 xt/30-backend-workers.t create mode 100644 xt/lib/App/NetdiscoX/Worker/Plugin/TestFive.pm create mode 100644 xt/lib/App/NetdiscoX/Worker/Plugin/TestFour.pm create mode 100644 xt/lib/App/NetdiscoX/Worker/Plugin/TestOne.pm create mode 100644 xt/lib/App/NetdiscoX/Worker/Plugin/TestThree.pm create mode 100644 xt/lib/App/NetdiscoX/Worker/Plugin/TestTwo.pm diff --git a/Build.PL b/Build.PL index 2de272fa..76357e2f 100644 --- a/Build.PL +++ b/Build.PL @@ -13,6 +13,9 @@ Module::Build->new( build_requires => { }, requires => { + 'aliased' => '0', + 'namespace::clean' => '0.24', + 'version' => '0.9902', 'Algorithm::Cron' => '0.07', 'AnyEvent' => '7.05', 'AnyEvent::DNS::EtcHosts' => '0', @@ -40,6 +43,7 @@ Module::Build->new( 'JSON::XS' => '3.01', 'List::MoreUtils' => '0.33', 'MIME::Base64' => '3.13', + 'Module::Find' => '0.13', 'Module::Load' => '0.32', 'Moo' => '1.001000', 'MCE' => '1.703', @@ -57,6 +61,7 @@ Module::Build->new( 'Plack::Middleware::ReverseProxy' => '0.15', 'Pod::Usage' => 0, 'Role::Tiny' => '1.002005', + 'Scope::Guard' => 0, 'Sereal' => '0', 'Socket6' => '0.23', 'Starman' => '0.4008', @@ -72,8 +77,6 @@ Module::Build->new( 'URL::Encode' => '0.01', 'YAML' => '0.84', 'YAML::XS' => '0.41', - 'namespace::clean' => '0.24', - 'version' => '0.9902', }, recommends => { 'Graph' => '0', diff --git a/Changes b/Changes index 78cdf8e1..bc4c8c84 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,5 @@ +2.036012_001 - EXPERIMENTAL RELEASE + 2.036011 - 2017-10-09 [BUG FIXES] diff --git a/MANIFEST b/MANIFEST index a5fed041..260a35b1 100644 --- a/MANIFEST +++ b/MANIFEST @@ -16,24 +16,10 @@ Changes lib/App/Netdisco.pm lib/App/Netdisco/AnyEvent/Nbtstat.pm lib/App/Netdisco/Backend/Job.pm -lib/App/Netdisco/Backend/Util.pm -lib/App/Netdisco/Backend/Worker/Common.pm -lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm -lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm -lib/App/Netdisco/Backend/Worker/Manager.pm -lib/App/Netdisco/Backend/Worker/Poller.pm -lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm -lib/App/Netdisco/Backend/Worker/Poller/Common.pm -lib/App/Netdisco/Backend/Worker/Poller/Device.pm -lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm -lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm -lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm -lib/App/Netdisco/Backend/Worker/Scheduler.pm +lib/App/Netdisco/Backend/Role/Manager.pm +lib/App/Netdisco/Backend/Role/Poller.pm +lib/App/Netdisco/Backend/Role/Scheduler.pm lib/App/Netdisco/Configuration.pm -lib/App/Netdisco/Core/Arpnip.pm -lib/App/Netdisco/Core/Discover.pm -lib/App/Netdisco/Core/Macsuck.pm -lib/App/Netdisco/Core/Nbtstat.pm lib/App/Netdisco/DB.pm lib/App/Netdisco/DB/ExplicitLocking.pm lib/App/Netdisco/DB/Result/Admin.pm @@ -114,7 +100,8 @@ lib/App/Netdisco/Manual/ReleaseNotes.pod lib/App/Netdisco/Manual/Systemd.pod lib/App/Netdisco/Manual/Troubleshooting.pod lib/App/Netdisco/Manual/Vendors.pod -lib/App/Netdisco/Manual/WritingPlugins.pod +lib/App/Netdisco/Manual/WritingWebPlugins.pod +lib/App/Netdisco/Manual/WritingWorkers.pod lib/App/Netdisco/SSHCollector/Platform/ACE.pm lib/App/Netdisco/SSHCollector/Platform/ASA.pm lib/App/Netdisco/SSHCollector/Platform/BigIP.pm @@ -126,12 +113,14 @@ lib/App/Netdisco/SSHCollector/Platform/IOSXR.pm lib/App/Netdisco/SSHCollector/Platform/Linux.pm lib/App/Netdisco/SSHCollector/Platform/NXOS.pm lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm -lib/App/Netdisco/Util/Backend.pm +lib/App/Netdisco/Transport/SNMP.pm lib/App/Netdisco/Util/Device.pm lib/App/Netdisco/Util/DNS.pm lib/App/Netdisco/Util/ExpandParams.pm lib/App/Netdisco/Util/FastResolver.pm lib/App/Netdisco/Util/Graph.pm +lib/App/Netdisco/Util/MCE.pm +lib/App/Netdisco/Util/Nbtstat.pm lib/App/Netdisco/Util/Node.pm lib/App/Netdisco/Util/NodeMonitor.pm lib/App/Netdisco/Util/Noop.pm @@ -203,6 +192,49 @@ lib/App/Netdisco/Web/Search.pm lib/App/Netdisco/Web/Static.pm lib/App/Netdisco/Web/Statistics.pm lib/App/Netdisco/Web/TypeAhead.pm +lib/App/Netdisco/Worker.pm +lib/App/Netdisco/Worker/Plugin.pm +lib/App/Netdisco/Worker/Plugin/Arpnip.pm +lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm +lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm +lib/App/Netdisco/Worker/Plugin/Arpwalk.pm +lib/App/Netdisco/Worker/Plugin/Contact.pm +lib/App/Netdisco/Worker/Plugin/Delete.pm +lib/App/Netdisco/Worker/Plugin/Discover.pm +lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm +lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm +lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm +lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm +lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm +lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm +lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm +lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm +lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm +lib/App/Netdisco/Worker/Plugin/Expire.pm +lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm +lib/App/Netdisco/Worker/Plugin/Graph.pm +lib/App/Netdisco/Worker/Plugin/Location.pm +lib/App/Netdisco/Worker/Plugin/Macsuck.pm +lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm +lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm +lib/App/Netdisco/Worker/Plugin/Macwalk.pm +lib/App/Netdisco/Worker/Plugin/Monitor.pm +lib/App/Netdisco/Worker/Plugin/Nbtstat.pm +lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm +lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm +lib/App/Netdisco/Worker/Plugin/PortControl.pm +lib/App/Netdisco/Worker/Plugin/PortName.pm +lib/App/Netdisco/Worker/Plugin/Power.pm +lib/App/Netdisco/Worker/Plugin/Psql.pm +lib/App/Netdisco/Worker/Plugin/Renumber.pm +lib/App/Netdisco/Worker/Plugin/Show.pm +lib/App/Netdisco/Worker/Plugin/Stats.pm +lib/App/Netdisco/Worker/Plugin/Test.pm +lib/App/Netdisco/Worker/Plugin/Test/Core.pm +lib/App/Netdisco/Worker/Plugin/Vlan.pm +lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm +lib/App/Netdisco/Worker/Runner.pm +lib/App/Netdisco/Worker/Status.pm lib/Dancer/Template/NetdiscoTemplateToolkit.pm LICENCE MANIFEST This list of files diff --git a/META.json b/META.json index 4e25919a..2dfe96de 100644 --- a/META.json +++ b/META.json @@ -56,6 +56,7 @@ "List::MoreUtils" : "0.33", "MCE" : "1.703", "MIME::Base64" : "3.13", + "Module::Find" : "0.13", "Module::Load" : "0.32", "Moo" : "1.001000", "Net::DNS" : "0.72", @@ -74,6 +75,7 @@ "Role::Tiny" : "1.002005", "SNMP::Info" : "3.37", "SQL::Translator" : "0.11018", + "Scope::Guard" : "0", "Sereal" : "0", "Socket6" : "0.23", "Starman" : "0.4008", @@ -87,6 +89,7 @@ "URL::Encode" : "0.01", "YAML" : "0.84", "YAML::XS" : "0.41", + "aliased" : "0", "namespace::clean" : "0.24", "version" : "0.9902" } @@ -101,7 +104,7 @@ "provides" : { "App::Netdisco" : { "file" : "lib/App/Netdisco.pm", - "version" : "2.036011" + "version" : "2.036012_001" }, "App::Netdisco::AnyEvent::Nbtstat" : { "file" : "lib/App/Netdisco/AnyEvent/Nbtstat.pm" @@ -109,60 +112,18 @@ "App::Netdisco::Backend::Job" : { "file" : "lib/App/Netdisco/Backend/Job.pm" }, - "App::Netdisco::Backend::Util" : { - "file" : "lib/App/Netdisco/Backend/Util.pm" + "App::Netdisco::Backend::Role::Manager" : { + "file" : "lib/App/Netdisco/Backend/Role/Manager.pm" }, - "App::Netdisco::Backend::Worker::Common" : { - "file" : "lib/App/Netdisco/Backend/Worker/Common.pm" + "App::Netdisco::Backend::Role::Poller" : { + "file" : "lib/App/Netdisco/Backend/Role/Poller.pm" }, - "App::Netdisco::Backend::Worker::Interactive::DeviceActions" : { - "file" : "lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm" - }, - "App::Netdisco::Backend::Worker::Interactive::PortActions" : { - "file" : "lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm" - }, - "App::Netdisco::Backend::Worker::Manager" : { - "file" : "lib/App/Netdisco/Backend/Worker/Manager.pm" - }, - "App::Netdisco::Backend::Worker::Poller" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Arpnip" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Common" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Common.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Device" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Device.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Expiry" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Macsuck" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm" - }, - "App::Netdisco::Backend::Worker::Poller::Nbtstat" : { - "file" : "lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm" - }, - "App::Netdisco::Backend::Worker::Scheduler" : { - "file" : "lib/App/Netdisco/Backend/Worker/Scheduler.pm" + "App::Netdisco::Backend::Role::Scheduler" : { + "file" : "lib/App/Netdisco/Backend/Role/Scheduler.pm" }, "App::Netdisco::Configuration" : { "file" : "lib/App/Netdisco/Configuration.pm" }, - "App::Netdisco::Core::Arpnip" : { - "file" : "lib/App/Netdisco/Core/Arpnip.pm" - }, - "App::Netdisco::Core::Discover" : { - "file" : "lib/App/Netdisco/Core/Discover.pm" - }, - "App::Netdisco::Core::Macsuck" : { - "file" : "lib/App/Netdisco/Core/Macsuck.pm" - }, - "App::Netdisco::Core::Nbtstat" : { - "file" : "lib/App/Netdisco/Core/Nbtstat.pm" - }, "App::Netdisco::DB" : { "file" : "lib/App/Netdisco/DB.pm", "version" : "44" @@ -413,8 +374,8 @@ "App::Netdisco::SSHCollector::Platform::PaloAlto" : { "file" : "lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm" }, - "App::Netdisco::Util::Backend" : { - "file" : "lib/App/Netdisco/Util/Backend.pm" + "App::Netdisco::Transport::SNMP" : { + "file" : "lib/App/Netdisco/Transport/SNMP.pm" }, "App::Netdisco::Util::DNS" : { "file" : "lib/App/Netdisco/Util/DNS.pm" @@ -431,6 +392,12 @@ "App::Netdisco::Util::Graph" : { "file" : "lib/App/Netdisco/Util/Graph.pm" }, + "App::Netdisco::Util::MCE" : { + "file" : "lib/App/Netdisco/Util/MCE.pm" + }, + "App::Netdisco::Util::Nbtstat" : { + "file" : "lib/App/Netdisco/Util/Nbtstat.pm" + }, "App::Netdisco::Util::Node" : { "file" : "lib/App/Netdisco/Util/Node.pm" }, @@ -644,18 +611,147 @@ "App::Netdisco::Web::TypeAhead" : { "file" : "lib/App/Netdisco/Web/TypeAhead.pm" }, + "App::Netdisco::Worker" : { + "file" : "lib/App/Netdisco/Worker.pm" + }, + "App::Netdisco::Worker::Plugin" : { + "file" : "lib/App/Netdisco/Worker/Plugin.pm" + }, + "App::Netdisco::Worker::Plugin::Arpnip" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Arpnip.pm" + }, + "App::Netdisco::Worker::Plugin::Arpnip::Nodes" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm" + }, + "App::Netdisco::Worker::Plugin::Arpnip::Subnets" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm" + }, + "App::Netdisco::Worker::Plugin::Arpwalk" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Arpwalk.pm" + }, + "App::Netdisco::Worker::Plugin::Contact" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Contact.pm" + }, + "App::Netdisco::Worker::Plugin::Delete" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Delete.pm" + }, + "App::Netdisco::Worker::Plugin::Discover" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::CanonicalIP" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::Entities" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::Neighbors" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::PortPower" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::Properties" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::VLANs" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::Wireless" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm" + }, + "App::Netdisco::Worker::Plugin::Discover::WithNodes" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm" + }, + "App::Netdisco::Worker::Plugin::DiscoverAll" : { + "file" : "lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm" + }, + "App::Netdisco::Worker::Plugin::Expire" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Expire.pm" + }, + "App::Netdisco::Worker::Plugin::ExpireNodes" : { + "file" : "lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm" + }, + "App::Netdisco::Worker::Plugin::Graph" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Graph.pm" + }, + "App::Netdisco::Worker::Plugin::Location" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Location.pm" + }, + "App::Netdisco::Worker::Plugin::Macsuck" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Macsuck.pm" + }, + "App::Netdisco::Worker::Plugin::Macsuck::Nodes" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm" + }, + "App::Netdisco::Worker::Plugin::Macsuck::WirelessNodes" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm" + }, + "App::Netdisco::Worker::Plugin::Macwalk" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Macwalk.pm" + }, + "App::Netdisco::Worker::Plugin::Monitor" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Monitor.pm" + }, + "App::Netdisco::Worker::Plugin::Nbtstat" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Nbtstat.pm" + }, + "App::Netdisco::Worker::Plugin::Nbtstat::Core" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm" + }, + "App::Netdisco::Worker::Plugin::Nbtwalk" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm" + }, + "App::Netdisco::Worker::Plugin::PortControl" : { + "file" : "lib/App/Netdisco/Worker/Plugin/PortControl.pm" + }, + "App::Netdisco::Worker::Plugin::PortName" : { + "file" : "lib/App/Netdisco/Worker/Plugin/PortName.pm" + }, + "App::Netdisco::Worker::Plugin::Power" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Power.pm" + }, + "App::Netdisco::Worker::Plugin::Psql" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Psql.pm" + }, + "App::Netdisco::Worker::Plugin::Renumber" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Renumber.pm" + }, + "App::Netdisco::Worker::Plugin::Show" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Show.pm" + }, + "App::Netdisco::Worker::Plugin::Stats" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Stats.pm" + }, + "App::Netdisco::Worker::Plugin::Test" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Test.pm" + }, + "App::Netdisco::Worker::Plugin::Test::Core" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Test/Core.pm" + }, + "App::Netdisco::Worker::Plugin::Vlan" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Vlan.pm" + }, + "App::Netdisco::Worker::Plugin::Vlan::Native" : { + "file" : "lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm" + }, + "App::Netdisco::Worker::Runner" : { + "file" : "lib/App/Netdisco/Worker/Runner.pm" + }, + "App::Netdisco::Worker::Status" : { + "file" : "lib/App/Netdisco/Worker/Status.pm" + }, "Dancer::Template::NetdiscoTemplateToolkit" : { "file" : "lib/Dancer/Template/NetdiscoTemplateToolkit.pm" } }, - "release_status" : "stable", + "release_status" : "testing", "resources" : { "bugtracker" : { "web" : "https://github.com/netdisco/netdisco/issues" }, "homepage" : "http://netdisco.org/", "license" : [ - "http://opensource.org/licenses/bsd-license.php" + "http://opensource.org/licenses/BSD-3-Clause" ], "repository" : { "url" : "https://github.com/netdisco/netdisco" @@ -663,6 +759,6 @@ "x_IRC" : "irc://irc.freenode.org/#netdisco", "x_MailingList" : "https://lists.sourceforge.net/lists/listinfo/netdisco-users" }, - "version" : "2.036011", + "version" : "2.036012_001", "x_serialization_backend" : "JSON::PP version 2.94" } diff --git a/META.yml b/META.yml index 9dd7b8d3..98df76ea 100644 --- a/META.yml +++ b/META.yml @@ -18,47 +18,19 @@ name: App-Netdisco provides: App::Netdisco: file: lib/App/Netdisco.pm - version: '2.036011' + version: 2.036012_001 App::Netdisco::AnyEvent::Nbtstat: file: lib/App/Netdisco/AnyEvent/Nbtstat.pm App::Netdisco::Backend::Job: file: lib/App/Netdisco/Backend/Job.pm - App::Netdisco::Backend::Util: - file: lib/App/Netdisco/Backend/Util.pm - App::Netdisco::Backend::Worker::Common: - file: lib/App/Netdisco/Backend/Worker/Common.pm - App::Netdisco::Backend::Worker::Interactive::DeviceActions: - file: lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm - App::Netdisco::Backend::Worker::Interactive::PortActions: - file: lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm - App::Netdisco::Backend::Worker::Manager: - file: lib/App/Netdisco/Backend/Worker/Manager.pm - App::Netdisco::Backend::Worker::Poller: - file: lib/App/Netdisco/Backend/Worker/Poller.pm - App::Netdisco::Backend::Worker::Poller::Arpnip: - file: lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm - App::Netdisco::Backend::Worker::Poller::Common: - file: lib/App/Netdisco/Backend/Worker/Poller/Common.pm - App::Netdisco::Backend::Worker::Poller::Device: - file: lib/App/Netdisco/Backend/Worker/Poller/Device.pm - App::Netdisco::Backend::Worker::Poller::Expiry: - file: lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm - App::Netdisco::Backend::Worker::Poller::Macsuck: - file: lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm - App::Netdisco::Backend::Worker::Poller::Nbtstat: - file: lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm - App::Netdisco::Backend::Worker::Scheduler: - file: lib/App/Netdisco/Backend/Worker/Scheduler.pm + App::Netdisco::Backend::Role::Manager: + file: lib/App/Netdisco/Backend/Role/Manager.pm + App::Netdisco::Backend::Role::Poller: + file: lib/App/Netdisco/Backend/Role/Poller.pm + App::Netdisco::Backend::Role::Scheduler: + file: lib/App/Netdisco/Backend/Role/Scheduler.pm App::Netdisco::Configuration: file: lib/App/Netdisco/Configuration.pm - App::Netdisco::Core::Arpnip: - file: lib/App/Netdisco/Core/Arpnip.pm - App::Netdisco::Core::Discover: - file: lib/App/Netdisco/Core/Discover.pm - App::Netdisco::Core::Macsuck: - file: lib/App/Netdisco/Core/Macsuck.pm - App::Netdisco::Core::Nbtstat: - file: lib/App/Netdisco/Core/Nbtstat.pm App::Netdisco::DB: file: lib/App/Netdisco/DB.pm version: '44' @@ -226,8 +198,8 @@ provides: file: lib/App/Netdisco/SSHCollector/Platform/NXOS.pm App::Netdisco::SSHCollector::Platform::PaloAlto: file: lib/App/Netdisco/SSHCollector/Platform/PaloAlto.pm - App::Netdisco::Util::Backend: - file: lib/App/Netdisco/Util/Backend.pm + App::Netdisco::Transport::SNMP: + file: lib/App/Netdisco/Transport/SNMP.pm App::Netdisco::Util::DNS: file: lib/App/Netdisco/Util/DNS.pm App::Netdisco::Util::Device: @@ -238,6 +210,10 @@ provides: file: lib/App/Netdisco/Util/FastResolver.pm App::Netdisco::Util::Graph: file: lib/App/Netdisco/Util/Graph.pm + App::Netdisco::Util::MCE: + file: lib/App/Netdisco/Util/MCE.pm + App::Netdisco::Util::Nbtstat: + file: lib/App/Netdisco/Util/Nbtstat.pm App::Netdisco::Util::Node: file: lib/App/Netdisco/Util/Node.pm App::Netdisco::Util::NodeMonitor: @@ -380,6 +356,92 @@ provides: file: lib/App/Netdisco/Web/Statistics.pm App::Netdisco::Web::TypeAhead: file: lib/App/Netdisco/Web/TypeAhead.pm + App::Netdisco::Worker: + file: lib/App/Netdisco/Worker.pm + App::Netdisco::Worker::Plugin: + file: lib/App/Netdisco/Worker/Plugin.pm + App::Netdisco::Worker::Plugin::Arpnip: + file: lib/App/Netdisco/Worker/Plugin/Arpnip.pm + App::Netdisco::Worker::Plugin::Arpnip::Nodes: + file: lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm + App::Netdisco::Worker::Plugin::Arpnip::Subnets: + file: lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm + App::Netdisco::Worker::Plugin::Arpwalk: + file: lib/App/Netdisco/Worker/Plugin/Arpwalk.pm + App::Netdisco::Worker::Plugin::Contact: + file: lib/App/Netdisco/Worker/Plugin/Contact.pm + App::Netdisco::Worker::Plugin::Delete: + file: lib/App/Netdisco/Worker/Plugin/Delete.pm + App::Netdisco::Worker::Plugin::Discover: + file: lib/App/Netdisco/Worker/Plugin/Discover.pm + App::Netdisco::Worker::Plugin::Discover::CanonicalIP: + file: lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm + App::Netdisco::Worker::Plugin::Discover::Entities: + file: lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm + App::Netdisco::Worker::Plugin::Discover::Neighbors: + file: lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm + App::Netdisco::Worker::Plugin::Discover::PortPower: + file: lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm + App::Netdisco::Worker::Plugin::Discover::Properties: + file: lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm + App::Netdisco::Worker::Plugin::Discover::VLANs: + file: lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm + App::Netdisco::Worker::Plugin::Discover::Wireless: + file: lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm + App::Netdisco::Worker::Plugin::Discover::WithNodes: + file: lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm + App::Netdisco::Worker::Plugin::DiscoverAll: + file: lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm + App::Netdisco::Worker::Plugin::Expire: + file: lib/App/Netdisco/Worker/Plugin/Expire.pm + App::Netdisco::Worker::Plugin::ExpireNodes: + file: lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm + App::Netdisco::Worker::Plugin::Graph: + file: lib/App/Netdisco/Worker/Plugin/Graph.pm + App::Netdisco::Worker::Plugin::Location: + file: lib/App/Netdisco/Worker/Plugin/Location.pm + App::Netdisco::Worker::Plugin::Macsuck: + file: lib/App/Netdisco/Worker/Plugin/Macsuck.pm + App::Netdisco::Worker::Plugin::Macsuck::Nodes: + file: lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm + App::Netdisco::Worker::Plugin::Macsuck::WirelessNodes: + file: lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm + App::Netdisco::Worker::Plugin::Macwalk: + file: lib/App/Netdisco/Worker/Plugin/Macwalk.pm + App::Netdisco::Worker::Plugin::Monitor: + file: lib/App/Netdisco/Worker/Plugin/Monitor.pm + App::Netdisco::Worker::Plugin::Nbtstat: + file: lib/App/Netdisco/Worker/Plugin/Nbtstat.pm + App::Netdisco::Worker::Plugin::Nbtstat::Core: + file: lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm + App::Netdisco::Worker::Plugin::Nbtwalk: + file: lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm + App::Netdisco::Worker::Plugin::PortControl: + file: lib/App/Netdisco/Worker/Plugin/PortControl.pm + App::Netdisco::Worker::Plugin::PortName: + file: lib/App/Netdisco/Worker/Plugin/PortName.pm + App::Netdisco::Worker::Plugin::Power: + file: lib/App/Netdisco/Worker/Plugin/Power.pm + App::Netdisco::Worker::Plugin::Psql: + file: lib/App/Netdisco/Worker/Plugin/Psql.pm + App::Netdisco::Worker::Plugin::Renumber: + file: lib/App/Netdisco/Worker/Plugin/Renumber.pm + App::Netdisco::Worker::Plugin::Show: + file: lib/App/Netdisco/Worker/Plugin/Show.pm + App::Netdisco::Worker::Plugin::Stats: + file: lib/App/Netdisco/Worker/Plugin/Stats.pm + App::Netdisco::Worker::Plugin::Test: + file: lib/App/Netdisco/Worker/Plugin/Test.pm + App::Netdisco::Worker::Plugin::Test::Core: + file: lib/App/Netdisco/Worker/Plugin/Test/Core.pm + App::Netdisco::Worker::Plugin::Vlan: + file: lib/App/Netdisco/Worker/Plugin/Vlan.pm + App::Netdisco::Worker::Plugin::Vlan::Native: + file: lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm + App::Netdisco::Worker::Runner: + file: lib/App/Netdisco/Worker/Runner.pm + App::Netdisco::Worker::Status: + file: lib/App/Netdisco/Worker/Status.pm Dancer::Template::NetdiscoTemplateToolkit: file: lib/Dancer/Template/NetdiscoTemplateToolkit.pm recommends: @@ -416,6 +478,7 @@ requires: List::MoreUtils: '0.33' MCE: '1.703' MIME::Base64: '3.13' + Module::Find: '0.13' Module::Load: '0.32' Moo: '1.001000' Net::DNS: '0.72' @@ -434,6 +497,7 @@ requires: Role::Tiny: '1.002005' SNMP::Info: '3.37' SQL::Translator: '0.11018' + Scope::Guard: '0' Sereal: '0' Socket6: '0.23' Starman: '0.4008' @@ -447,6 +511,7 @@ requires: URL::Encode: '0.01' YAML: '0.84' YAML::XS: '0.41' + aliased: '0' namespace::clean: '0.24' version: '0.9902' resources: @@ -454,7 +519,7 @@ resources: MailingList: https://lists.sourceforge.net/lists/listinfo/netdisco-users bugtracker: https://github.com/netdisco/netdisco/issues homepage: http://netdisco.org/ - license: http://opensource.org/licenses/bsd-license.php + license: http://opensource.org/licenses/BSD-3-Clause repository: https://github.com/netdisco/netdisco -version: '2.036011' +version: 2.036012_001 x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff --git a/bin/netdisco-backend-fg b/bin/netdisco-backend-fg index d89fc9e4..31e2764d 100755 --- a/bin/netdisco-backend-fg +++ b/bin/netdisco-backend-fg @@ -29,26 +29,10 @@ BEGIN { setting('workers')->{'BACKEND'} ||= (hostfqdn || 'fqdn-undefined'); } -use App::Netdisco::Util::Backend; +use App::Netdisco::Util::MCE; # set $0 and parse maxworkers use NetAddr::IP::Lite ':lower'; # to quench AF_INET6 symbol errors use Role::Tiny::With; -# preload all worker modules into shared memory -use App::Netdisco::Backend::Job (); -use App::Netdisco::Backend::Util (); -use App::Netdisco::Backend::Worker::Common (); -use App::Netdisco::Backend::Worker::Interactive::DeviceActions (); -use App::Netdisco::Backend::Worker::Interactive::PortActions (); -use App::Netdisco::Backend::Worker::Manager (); -use App::Netdisco::Backend::Worker::Poller::Arpnip (); -use App::Netdisco::Backend::Worker::Poller::Common (); -use App::Netdisco::Backend::Worker::Poller::Device (); -use App::Netdisco::Backend::Worker::Poller::Expiry (); -use App::Netdisco::Backend::Worker::Poller::Macsuck (); -use App::Netdisco::Backend::Worker::Poller::Nbtstat (); -use App::Netdisco::Backend::Worker::Poller (); -use App::Netdisco::Backend::Worker::Scheduler (); - use MCE::Signal '-setpgrp'; use MCE::Flow Sereal => 1; use MCE::Queue; @@ -90,7 +74,7 @@ sub _mk_wkr { # post-fork, become manager, scheduler, poller, etc Role::Tiny->apply_roles_to_object( - $self => "App::Netdisco::Backend::Worker::$role"); + $self => "App::Netdisco::Backend::Role::$role"); $self->worker_begin if $self->can('worker_begin'); $self->worker_body; diff --git a/bin/netdisco-do b/bin/netdisco-do index bc1dbde2..1c5b0b43 100755 --- a/bin/netdisco-do +++ b/bin/netdisco-do @@ -35,17 +35,18 @@ BEGIN { # for netdisco app config use App::Netdisco; -use App::Netdisco::Backend::Job; use Dancer qw/:moose :script/; info "App::Netdisco version $App::Netdisco::VERSION loaded."; -use NetAddr::IP qw/:rfc3021 :lower/; -use App::Netdisco::Util::Device 'get_device'; - use Try::Tiny; use Pod::Usage; use Scalar::Util 'blessed'; +use NetAddr::IP qw/:rfc3021 :lower/; + +use App::Netdisco::Backend::Job; +use App::Netdisco::Util::Device 'get_device'; + use Getopt::Long; Getopt::Long::Configure ("bundling"); @@ -89,139 +90,14 @@ unless ($action) { ); } -# create worker (placeholder object for the role methods) +# create worker (placeholder object for the action runner) { package MyWorker; - use Moo; - use Module::Load (); - use Data::Printer (); - use Scalar::Util 'blessed'; - use NetAddr::IP qw/:rfc3021 :lower/; - use Dancer ':script'; - - use App::Netdisco::Util::SNMP (); - use App::Netdisco::Util::Device - qw/get_device delete_device renumber_device/; - - with 'App::Netdisco::Backend::Worker::Poller::Device'; - with 'App::Netdisco::Backend::Worker::Poller::Arpnip'; - with 'App::Netdisco::Backend::Worker::Poller::Macsuck'; - with 'App::Netdisco::Backend::Worker::Poller::Nbtstat'; - with 'App::Netdisco::Backend::Worker::Poller::Expiry'; - with 'App::Netdisco::Backend::Worker::Interactive::DeviceActions'; - with 'App::Netdisco::Backend::Worker::Interactive::PortActions'; - - eval { Module::Load::load 'App::Netdisco::Util::Graph' }; - sub graph { - App::Netdisco::Util::Graph::graph(); - return ('done', 'Generated graph data.'); - } - - use App::Netdisco::Util::NodeMonitor (); - sub monitor { - App::Netdisco::Util::NodeMonitor::monitor(); - return ('done', 'Generated monitor data.'); - } - - use App::Netdisco::Util::Statistics (); - sub stats { - App::Netdisco::Util::Statistics::update_stats(); - return ('done', 'Updated statistics.'); - } - - sub show { - my ($self, $job) = @_; - my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; - return ('error', 'Missing device (-d).') if !defined $device; - - $extra ||= 'interfaces'; my $class = undef; - ($class, $extra) = split(/::([^:]+)$/, $extra); - if ($class and $extra) { - $class = 'SNMP::Info::'.$class; - } - else { - $extra = $class; - undef $class; - } - my $i = App::Netdisco::Util::SNMP::snmp_connect($device, $class); - Data::Printer::p($i->$extra); - return ('done', sprintf "Showed %s response from %s.", $extra, $device->ip); - } - - sub delete { - my ($self, $job) = @_; - my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; - return ('error', 'Missing device (-d).') if !defined $device; - - $port = ($port ? 1 : 0); - delete_device($device, $port, $extra); - return ('done', sprintf "Deleted device %s.", $device->ip); - } - - sub renumber { - my ($self, $job) = @_; - my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; - return ('error', 'Missing device (-d).') if !defined $device; - my $old_ip = $device->ip; - - my $new_ip = NetAddr::IP->new($extra); - unless ($new_ip and $new_ip->addr ne '0.0.0.0') { - return ('error', "Bad host or IP: ".($extra || '0.0.0.0')); - } - - my $new_dev = get_device($new_ip->addr); - if ($new_dev and $new_dev->in_storage and ($new_dev->ip ne $device->ip)) { - return ('error', sprintf "Already know new device as: %s.", $new_dev->ip); - } - - renumber_device($device, $new_ip); - return ('done', sprintf 'Renumbered device %s to %s (%s).', - $device->ip, $new_ip, ($device->dns || '')); - } - - sub psql { - my ($self, $job) = @_; - my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; - - my $name = ($ENV{NETDISCO_DBNAME} || setting('database')->{name} || 'netdisco'); - my $host = setting('database')->{host}; - my $user = setting('database')->{user}; - my $pass = setting('database')->{pass}; - - my $portnum = undef; - if ($host and $host =~ m/([^;]+);port=(\d+)/) { - $host = $1; - $portnum = $2; - } - - $ENV{PGHOST} = $host if $host; - $ENV{PGPORT} = $portnum if defined $portnum; - $ENV{PGDATABASE} = $name; - $ENV{PGUSER} = $user; - $ENV{PGPASSWORD} = $pass; - $ENV{PGCLIENTENCODING} = 'UTF8'; - - if ($extra) { - system('psql', '-c', $extra); - } - else { - system('psql'); - } - return ('done', "psql session closed."); - } + with 'App::Netdisco::Worker::Runner'; } my $worker = MyWorker->new(); -# belt and braces check before we go ahead -if (not $worker->can( $action )) { - pod2usage( - -msg => (sprintf 'error: %s is not a valid action', $action), - -verbose => 2, - -exitval => 3, - ); -} - my $net = NetAddr::IP->new($device); if ($device and (!$net or $net->num == 0 or $net->addr eq '0.0.0.0')) { info sprintf '%s: error - Bad host, IP or prefix: %s', $action, $device; @@ -249,25 +125,31 @@ foreach my $host (@hostlist) { my $actiontext = ( ($job->device ? ('['.$job->device->ip.']') : '') . - ($job->action eq 'show' ? ('/'.$job->subaction) : '') + ($job->action eq 'show' ? ('/'. ($job->subaction || 'interfaces')) : '') ); # do job - my ($status, $log); try { - info sprintf '%s: %s started at %s', $action, $actiontext, scalar localtime; - ($status, $log) = $worker->$action($job); + $worker->run($job); } catch { - $status = 'error'; - $log = "error running job: $_"; + $job->status('error'); + $job->log("error running job: $_"); }; + if ($job->log eq 'no worker succeeded during main phase') { + pod2usage( + -msg => (sprintf 'error: %s is not a valid action', $action), + -verbose => 2, + -exitval => 3, + ); + } + info sprintf '%s: finished at %s', $action, scalar localtime; - info sprintf '%s: status %s: %s', $action, $status, $log; - $exitstatus = 1 if !defined $status or $status eq 'error'; + info sprintf '%s: status %s: %s', $action, $job->status, $job->log; + $exitstatus = 1 if !$exitstatus and $job->status ne 'done'; } exit $exitstatus; @@ -394,6 +276,10 @@ leaf with the class short name, for example "C" or ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e interfaces ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e Layer2::HP::interfaces +A paramter may be passed to the C method in the C<-p> parameter: + + ~netdisco/bin/netdisco-do show -d 192.0.2.1 -e has_layer -p 3 + =head2 psql Start an interactive terminal with the Netdisco PostgreSQL database. If you diff --git a/lib/App/Netdisco.pm b/lib/App/Netdisco.pm index d6d96173..a7952d94 100644 --- a/lib/App/Netdisco.pm +++ b/lib/App/Netdisco.pm @@ -4,7 +4,7 @@ use strict; use warnings; use 5.010_000; -our $VERSION = '2.036011'; +our $VERSION = '2.036012_001'; use App::Netdisco::Configuration; =head1 NAME diff --git a/lib/App/Netdisco/Backend/Job.pm b/lib/App/Netdisco/Backend/Job.pm index b4c27ccc..7811b53f 100644 --- a/lib/App/Netdisco/Backend/Job.pm +++ b/lib/App/Netdisco/Backend/Job.pm @@ -1,5 +1,8 @@ package App::Netdisco::Backend::Job; +use Dancer qw/:moose :syntax !error/; +use aliased 'App::Netdisco::Worker::Status'; + use Moo; use namespace::clean; @@ -16,7 +19,10 @@ foreach my $slot (qw/ username userip log - debug + + _current_phase + _last_namespace + _last_priority /) { has $slot => ( @@ -24,25 +30,147 @@ foreach my $slot (qw/ ); } +has '_statuslist' => ( + is => 'rw', + default => sub { [] }, +); + =head1 METHODS =head2 summary -An attempt to make a meaningful statement about the job. +An attempt to make a meaningful written statement about the job. =cut sub summary { - my $job = shift; - return join ' ', - $job->action, - ($job->device || ''), - ($job->port || ''); -# ($job->subaction ? (q{'}. $job->subaction .q{'}) : ''); + my $job = shift; + return join ' ', + $job->action, + ($job->device || ''), + ($job->port || ''); +} + +=head2 finalise_status + +Find the best status and log it into the job's C and C slots. + +The process is to track back from the last worker and find the best status, +which is C in early or main phases, or else any status in any non-user +phase. + +=cut + +sub finalise_status { + my $job = shift; + # use DDP; p $job->_statuslist; + + # fallback + $job->status('error'); + $job->log('failed to report from any worker!'); + + my $max_level = Status->error()->level; + + foreach my $status (reverse @{ $job->_statuslist }) { + next if $status->phase + and $status->phase !~ m/^(?:check|early|main)$/; + + next if $status->phase eq 'check' + and $status->level eq Status->done()->level; + + if ($status->level >= $max_level) { + $job->status( $status->status ); + $job->log( $status->log ); + $max_level = $status->level; + } + } +} + +=head2 check_passed + +Returns true if at least one worker during the C phase flagged status +C. + +=cut + +sub check_passed { + my $job = shift; + return true if 0 == scalar @{ $job->_statuslist }; + + foreach my $status (@{ $job->_statuslist }) { + next unless $status->phase and $status->phase eq 'check'; + return true if $status->is_ok; + } + return false; +} + +=head2 namespace_passed( \%workerconf ) + +Returns true when, for the namespace specified in the given configuration, a +worker of a higher priority level has already succeeded. + +=cut + +sub namespace_passed { + my ($job, $workerconf) = @_; + + if ($job->_last_namespace) { + foreach my $status (@{ $job->_statuslist }) { + next unless ($status->phase and $status->phase eq $workerconf->{phase}) + and ($workerconf->{namespace} eq $job->_last_namespace) + and ($workerconf->{priority} < $job->_last_priority); + return true if $status->is_ok; + } + } + + $job->_last_namespace( $workerconf->{namespace} ); + $job->_last_priority( $workerconf->{priority} ); + return false; +} + +=head2 enter_phase( $phase ) + +Pass the name of the phase being entered. + +=cut + +sub enter_phase { + my ($job, $phase) = @_; + + $job->_current_phase( $phase ); + debug "=> running workers for phase: $phase"; + + $job->_last_namespace( undef ); + $job->_last_priority( undef ); +} + +=head2 add_status + +Passed an L will add it to this job's internal +status cache. Phase slot of the Status will be set to the current phase. + +=cut + +sub add_status { + my ($job, $status) = @_; + return unless ref $status eq 'App::Netdisco::Worker::Status'; + $status->phase( $job->_current_phase || '' ); + push @{ $job->_statuslist }, $status; + debug $status->log if $status->log + and (($status->phase eq 'check') + or ($status->level ne Status->done()->level)); } =head1 ADDITIONAL COLUMNS +=head2 id + +Alias for the C column. + +=cut + +sub id { (shift)->job } + =head2 extra Alias for the C column. @@ -51,4 +179,4 @@ Alias for the C column. sub extra { (shift)->subaction } -1; +true; diff --git a/lib/App/Netdisco/Backend/Worker/Manager.pm b/lib/App/Netdisco/Backend/Role/Manager.pm similarity index 94% rename from lib/App/Netdisco/Backend/Worker/Manager.pm rename to lib/App/Netdisco/Backend/Role/Manager.pm index ee17349c..25d13ba7 100644 --- a/lib/App/Netdisco/Backend/Worker/Manager.pm +++ b/lib/App/Netdisco/Backend/Role/Manager.pm @@ -1,16 +1,16 @@ -package App::Netdisco::Backend::Worker::Manager; +package App::Netdisco::Backend::Role::Manager; use Dancer qw/:moose :syntax :script/; use List::Util 'sum'; -use App::Netdisco::Util::Backend; - -use Role::Tiny; -use namespace::clean; +use App::Netdisco::Util::MCE; use App::Netdisco::JobQueue qw/jq_locked jq_getsome jq_getsomep jq_lock jq_warm_thrusters/; +use Role::Tiny; +use namespace::clean; + sub worker_begin { my $self = shift; my $wid = $self->wid; @@ -58,7 +58,7 @@ sub worker_body { # mark job as running next unless jq_lock($job); info sprintf "mgr (%s): job %s booked out for this processing node", - $wid, $job->job; + $wid, $job->id; # copy job to local queue $self->{queue}->enqueuep(100, $job); @@ -75,7 +75,7 @@ sub worker_body { # mark job as running next unless jq_lock($job); info sprintf "mgr (%s): job %s booked out for this processing node", - $wid, $job->job; + $wid, $job->id; # copy job to local queue $self->{queue}->enqueue($job); diff --git a/lib/App/Netdisco/Backend/Worker/Common.pm b/lib/App/Netdisco/Backend/Role/Poller.pm similarity index 76% rename from lib/App/Netdisco/Backend/Worker/Common.pm rename to lib/App/Netdisco/Backend/Role/Poller.pm index 5390a4db..ff689f52 100644 --- a/lib/App/Netdisco/Backend/Worker/Common.pm +++ b/lib/App/Netdisco/Backend/Role/Poller.pm @@ -1,15 +1,18 @@ -package App::Netdisco::Backend::Worker::Common; +package App::Netdisco::Backend::Role::Poller; use Dancer qw/:moose :syntax :script/; use Try::Tiny; -use App::Netdisco::Util::Backend; +use App::Netdisco::Util::MCE; + +use Time::HiRes 'sleep'; +use App::Netdisco::JobQueue qw/jq_defer jq_complete/; use Role::Tiny; use namespace::clean; -use Time::HiRes 'sleep'; -use App::Netdisco::JobQueue qw/jq_defer jq_complete/; +# add dispatch methods for poller tasks +with 'App::Netdisco::Worker::Runner'; sub worker_begin { (shift)->{started} = time } @@ -22,17 +25,14 @@ sub worker_body { my $job = $self->{queue}->dequeue(1); next unless defined $job; - my $action = $job->action; try { $job->started(scalar localtime); prctl sprintf 'nd2: #%s poll: #%s: %s', - $wid, $job->job, $job->summary; + $wid, $job->id, $job->summary; info sprintf "pol (%s): starting %s job(%s) at %s", - $wid, $action, $job->job, $job->started; - my ($status, $log) = $self->$action($job); - $job->status($status); - $job->log($log); + $wid, $job->action, $job->id, $job->started; + $self->run($job); } catch { $job->status('error'); @@ -51,7 +51,7 @@ sub close_job { my $now = scalar localtime; info sprintf "pol (%s): wrapping up %s job(%s) - status %s at %s", - $self->wid, $job->action, $job->job, $job->status, $now; + $self->wid, $job->action, $job->id, $job->status, $now; try { if ($job->status eq 'defer') { diff --git a/lib/App/Netdisco/Backend/Worker/Scheduler.pm b/lib/App/Netdisco/Backend/Role/Scheduler.pm similarity index 96% rename from lib/App/Netdisco/Backend/Worker/Scheduler.pm rename to lib/App/Netdisco/Backend/Role/Scheduler.pm index 10b44422..a62f3f22 100644 --- a/lib/App/Netdisco/Backend/Worker/Scheduler.pm +++ b/lib/App/Netdisco/Backend/Role/Scheduler.pm @@ -1,15 +1,15 @@ -package App::Netdisco::Backend::Worker::Scheduler; +package App::Netdisco::Backend::Role::Scheduler; use Dancer qw/:moose :syntax :script/; use Algorithm::Cron; -use App::Netdisco::Util::Backend; +use App::Netdisco::Util::MCE; + +use App::Netdisco::JobQueue qw/jq_insert/; use Role::Tiny; use namespace::clean; -use App::Netdisco::JobQueue qw/jq_insert/; - sub worker_begin { my $self = shift; my $wid = $self->wid; diff --git a/lib/App/Netdisco/Backend/Util.pm b/lib/App/Netdisco/Backend/Util.pm deleted file mode 100644 index a957d5e6..00000000 --- a/lib/App/Netdisco/Backend/Util.pm +++ /dev/null @@ -1,17 +0,0 @@ -package App::Netdisco::Backend::Util; - -use strict; -use warnings; - -# support utilities for Backend Actions - -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ job_done job_error job_defer /; -our %EXPORT_TAGS = (all => \@EXPORT_OK); - -sub job_done { return ('done', shift) } -sub job_error { return ('error', shift) } -sub job_defer { return ('defer', shift) } - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm b/lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm deleted file mode 100644 index 09d6a558..00000000 --- a/lib/App/Netdisco/Backend/Worker/Interactive/DeviceActions.pm +++ /dev/null @@ -1,50 +0,0 @@ -package App::Netdisco::Backend::Worker::Interactive::DeviceActions; - -use App::Netdisco::Util::SNMP 'snmp_connect_rw'; -use App::Netdisco::Util::Device 'get_device'; -use App::Netdisco::Backend::Util ':all'; - -use Role::Tiny; -use namespace::clean; - -sub location { - my ($self, $job) = @_; - return _set_device_generic($job->device, 'location', $job->subaction); -} - -sub contact { - my ($self, $job) = @_; - return _set_device_generic($job->device, 'contact', $job->subaction); -} - -sub _set_device_generic { - my ($ip, $slot, $data) = @_; - $data ||= ''; - - # snmp connect using rw community - my $info = snmp_connect_rw($ip) - or return job_defer("Failed to connect to device [$ip] to update $slot"); - - my $method = 'set_'. $slot; - my $rv = $info->$method($data); - - if (!defined $rv) { - return job_error(sprintf 'Failed to set %s on [%s]: %s', - $slot, $ip, ($info->error || '')); - } - - # confirm the set happened - $info->clear_cache; - my $new_data = ($info->$slot || ''); - if ($new_data ne $data) { - return job_error("Verify of $slot update failed on [$ip]: $new_data"); - } - - # update netdisco DB - my $device = get_device($ip); - $device->update({$slot => $data}); - - return job_done("Updated $slot on [$ip] to [$data]"); -} - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm b/lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm deleted file mode 100644 index 549fcb89..00000000 --- a/lib/App/Netdisco/Backend/Worker/Interactive/PortActions.pm +++ /dev/null @@ -1,159 +0,0 @@ -package App::Netdisco::Backend::Worker::Interactive::PortActions; - -use App::Netdisco::Util::Port ':all'; -use App::Netdisco::Util::SNMP 'snmp_connect_rw'; -use App::Netdisco::Util::Device 'get_device'; -use App::Netdisco::Backend::Util ':all'; - -use Role::Tiny; -use namespace::clean; - -sub portname { - my ($self, $job) = @_; - return _set_port_generic($job, 'alias', 'name'); -} - -sub portcontrol { - my ($self, $job) = @_; - - my $port = get_port($job->device, $job->port) - or return job_error(sprintf "Unknown port name [%s] on device [%s]", - $job->port, $job->device); - - my $reconfig_check = port_reconfig_check($port); - return job_error("Cannot alter port: $reconfig_check") - if $reconfig_check; - - # need to remove "-other" which appears for power/portcontrol - (my $sa = $job->subaction) =~ s/-\w+//; - $job->subaction($sa); - - if ($sa eq 'bounce') { - $job->subaction('down'); - my @stat = _set_port_generic($job, 'up_admin'); - return @stat if $stat[0] ne 'done'; - $job->subaction('up'); - return _set_port_generic($job, 'up_admin'); - } - else { - return _set_port_generic($job, 'up_admin'); - } -} - -sub vlan { - my ($self, $job) = @_; - - my $port = get_port($job->device, $job->port) - or return job_error(sprintf "Unknown port name [%s] on device [%s]", - $job->port, $job->device); - - my $port_reconfig_check = port_reconfig_check($port); - return job_error("Cannot alter port: $port_reconfig_check") - if $port_reconfig_check; - - my $vlan_reconfig_check = vlan_reconfig_check($port); - return job_error("Cannot alter vlan: $vlan_reconfig_check") - if $vlan_reconfig_check; - - my @stat = _set_port_generic($job, 'pvid'); # for Cisco trunk - return @stat if $stat[0] eq 'done'; - return _set_port_generic($job, 'vlan'); -} - -sub _set_port_generic { - my ($job, $slot, $column) = @_; - $column ||= $slot; - - my $device = get_device($job->device); - my $ip = $device->ip; - my $pn = $job->port; - my $data = $job->subaction; - - my $port = get_port($ip, $pn) - or return job_error("Unknown port name [$pn] on device [$ip]"); - - if ($device->vendor ne 'netdisco') { - # snmp connect using rw community - my $info = snmp_connect_rw($ip) - or return job_defer("Failed to connect to device [$ip] to control port"); - - my $iid = get_iid($info, $port) - or return job_error("Failed to get port ID for [$pn] from [$ip]"); - - my $method = 'set_i_'. $slot; - my $rv = $info->$method($data, $iid); - - if (!defined $rv) { - return job_error(sprintf 'Failed to set [%s] %s to [%s] on [%s]: %s', - $pn, $slot, $data, $ip, ($info->error || '')); - } - - # confirm the set happened - $info->clear_cache; - my $check_method = 'i_'. $slot; - my $state = ($info->$check_method($iid) || ''); - if (ref {} ne ref $state or $state->{$iid} ne $data) { - return job_error("Verify of [$pn] $slot failed on [$ip]"); - } - } - - # update netdisco DB - $port->update({$column => $data}); - - return job_done("Updated [$pn] $slot status on [$ip] to [$data]"); -} - -sub power { - my ($self, $job) = @_; - - my $port = get_port($job->device, $job->port) - or return job_error(sprintf "Unknown port name [%s] on device [%s]", - $job->port, $job->device); - - return job_error("No PoE service on port [%s] on device [%s]") - unless $port->power; - - my $reconfig_check = port_reconfig_check($port); - return job_error("Cannot alter port: $reconfig_check") - if $reconfig_check; - - my $device = get_device($job->device); - my $ip = $device->ip; - my $pn = $job->port; - - # munge data - (my $data = $job->subaction) =~ s/-\w+//; # remove -other - $data = 'true' if $data =~ m/^(on|yes|up)$/; - $data = 'false' if $data =~ m/^(off|no|down)$/; - - # snmp connect using rw community - my $info = snmp_connect_rw($ip) - or return job_defer("Failed to connect to device [$ip] to control power"); - - my $powerid = get_powerid($info, $port) - or return job_error("Failed to get power ID for [$pn] from [$ip]"); - - my $rv = $info->set_peth_port_admin($data, $powerid); - - if (!defined $rv) { - return job_error(sprintf 'Failed to set [%s] power to [%s] on [%s]: %s', - $pn, $data, $ip, ($info->error || '')); - } - - # confirm the set happened - $info->clear_cache; - my $state = ($info->peth_port_admin($powerid) || ''); - if (ref {} ne ref $state or $state->{$powerid} ne $data) { - return job_error("Verify of [$pn] power failed on [$ip]"); - } - - # update netdisco DB - $port->power->update({ - admin => $data, - status => ($data eq 'false' ? 'disabled' : 'searching'), - }); - - return job_done("Updated [$pn] power status on [$ip] to [$data]"); -} - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller.pm b/lib/App/Netdisco/Backend/Worker/Poller.pm deleted file mode 100644 index 1e62d61b..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller.pm +++ /dev/null @@ -1,18 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller; - -use Role::Tiny; -use namespace::clean; - -# main worker body -with 'App::Netdisco::Backend::Worker::Common'; - -# add dispatch methods for poller tasks -with 'App::Netdisco::Backend::Worker::Poller::Device', - 'App::Netdisco::Backend::Worker::Poller::Arpnip', - 'App::Netdisco::Backend::Worker::Poller::Macsuck', - 'App::Netdisco::Backend::Worker::Poller::Nbtstat', - 'App::Netdisco::Backend::Worker::Poller::Expiry', - 'App::Netdisco::Backend::Worker::Interactive::DeviceActions', - 'App::Netdisco::Backend::Worker::Interactive::PortActions'; - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm b/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm deleted file mode 100644 index fb2ce593..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller/Arpnip.pm +++ /dev/null @@ -1,18 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller::Arpnip; - -use App::Netdisco::Core::Arpnip 'do_arpnip'; -use App::Netdisco::Util::Device 'is_arpnipable_now'; - -use Role::Tiny; -use namespace::clean; - -with 'App::Netdisco::Backend::Worker::Poller::Common'; - -sub arpnip_action { \&do_arpnip } -sub arpnip_filter { \&is_arpnipable_now } -sub arpnip_layer { 3 } - -sub arpwalk { (shift)->_walk_body('arpnip', @_) } -sub arpnip { (shift)->_single_body('arpnip', @_) } - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Common.pm b/lib/App/Netdisco/Backend/Worker/Poller/Common.pm deleted file mode 100644 index 34533d64..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller/Common.pm +++ /dev/null @@ -1,99 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller::Common; - -use Dancer qw/:moose :syntax :script/; - -use App::Netdisco::Util::SNMP 'snmp_connect'; -use App::Netdisco::Util::Device 'get_device'; -use App::Netdisco::Backend::Util ':all'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; - -use Dancer::Plugin::DBIC 'schema'; -use NetAddr::IP::Lite ':lower'; - -use Role::Tiny; -use namespace::clean; - -# queue a job for all devices known to Netdisco -sub _walk_body { - my ($self, $job_type, $job) = @_; - - my $layer_method = $job_type .'_layer'; - my $job_layer = $self->$layer_method; - - my %queued = map {$_ => 1} jq_queued($job_type); - my @devices = schema('netdisco')->resultset('Device')->search({ - -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], - })->has_layer($job_layer)->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; - - jq_insert([ - map {{ - device => $_, - action => $job_type, - username => $job->username, - userip => $job->userip, - }} (@filtered_devices) - ]); - - return job_done("Queued $job_type job for all devices"); -} - -sub _single_body { - my ($self, $job_type, $job) = @_; - - my $action_method = $job_type .'_action'; - my $job_action = $self->$action_method; - - my $layer_method = $job_type .'_layer'; - my $job_layer = $self->$layer_method; - - my $device = get_device($job->device) - or job_error("$job_type failed: unable to interpret device parameter"); - my $host = $device->ip; - - if ($device->in_storage - and $device->vendor and $device->vendor eq 'netdisco') { - return job_done("$job_type skipped: $host is pseudo-device"); - } - - my $filter_method = $job_type .'_filter'; - my $job_filter = $self->$filter_method; - - unless ($job_filter->($device->ip)) { - return job_defer("$job_type deferred: $host is not ${job_type}able"); - } - - my $snmp = snmp_connect($device); - if (!defined $snmp) { - return job_defer("$job_type failed: could not SNMP connect to $host"); - } - - unless ($snmp->has_layer( $job_layer )) { - return job_done("Skipped $job_type for device $host without OSI layer $job_layer capability"); - } - - $job_action->($device, $snmp); - - return job_done("Ended $job_type for $host"); -} - -sub _single_node_body { - my ($self, $job_type, $node, $now) = @_; - - my $action_method = $job_type .'_action'; - my $job_action = $self->$action_method; - - my $filter_method = $job_type .'_filter'; - my $job_filter = $self->$filter_method; - - unless ($job_filter->($node)) { - return job_defer("$job_type deferred: $node is not ${job_type}able"); - } - - $job_action->($node, $now); - - # would be ignored if wrapped in a loop - return job_done("Ended $job_type for $node"); -} - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Device.pm b/lib/App/Netdisco/Backend/Worker/Poller/Device.pm deleted file mode 100644 index 95c6c49a..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller/Device.pm +++ /dev/null @@ -1,100 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller::Device; - -use Dancer qw/:moose :syntax :script/; - -use App::Netdisco::Util::SNMP 'snmp_connect'; -use App::Netdisco::Util::Device qw/get_device is_discoverable_now/; -use App::Netdisco::Core::Discover ':all'; -use App::Netdisco::Backend::Util ':all'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; - -use Dancer::Plugin::DBIC 'schema'; -use NetAddr::IP::Lite ':lower'; - -use Role::Tiny; -use namespace::clean; - -# queue a discover job for all devices known to Netdisco -sub discoverall { - my ($self, $job) = @_; - - my %queued = map {$_ => 1} jq_queued('discover'); - my @devices = schema('netdisco')->resultset('Device')->search({ - -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], - })->get_column('ip')->all; - my @filtered_devices = grep {!exists $queued{$_}} @devices; - - jq_insert([ - map {{ - device => $_, - action => 'discover', - username => $job->username, - userip => $job->userip, - }} (@filtered_devices) - ]); - - return job_done("Queued discover job for all devices"); -} - -# run a discover job for one device, and its *new* neighbors -sub discover { - my ($self, $job) = @_; - - my $device = get_device($job->device) - or return job_error( - "discover failed: unable to interpret device parameter: " - . ($job->device || "''")); - my $host = $device->ip; - - if ($device->ip eq '0.0.0.0') { - return job_error("discover failed: no device param (need -d ?)"); - } - - if ($device->in_storage - and $device->vendor and $device->vendor eq 'netdisco') { - return job_done("discover skipped: $host is pseudo-device"); - } - - unless (is_discoverable_now($device)) { - return job_defer("discover deferred: $host is not discoverable"); - } - - my $snmp = snmp_connect($device); - if (!defined $snmp) { - return job_defer("discover failed: could not SNMP connect to $host"); - } - - store_device($device, $snmp); - set_canonical_ip($device, $snmp); # must come after store_device - store_interfaces($device, $snmp); - store_wireless($device, $snmp); - store_vlans($device, $snmp); - store_power($device, $snmp); - store_modules($device, $snmp) if setting('store_modules'); - discover_new_neighbors($device, $snmp); - - # if requested, and the device has not yet been arpniped/macsucked, queue now - if ($device->in_storage and $job->subaction and $job->subaction eq 'with-nodes') { - if (!defined $device->last_macsuck) { - jq_insert({ - device => $device->ip, - action => 'macsuck', - username => $job->username, - userip => $job->userip, - }); - } - - if (!defined $device->last_arpnip) { - jq_insert({ - device => $device->ip, - action => 'arpnip', - username => $job->username, - userip => $job->userip, - }); - } - } - - return job_done("Ended discover for $host"); -} - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm b/lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm deleted file mode 100644 index bb927a31..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller/Macsuck.pm +++ /dev/null @@ -1,18 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller::Macsuck; - -use App::Netdisco::Core::Macsuck 'do_macsuck'; -use App::Netdisco::Util::Device 'is_macsuckable_now'; - -use Role::Tiny; -use namespace::clean; - -with 'App::Netdisco::Backend::Worker::Poller::Common'; - -sub macsuck_action { \&do_macsuck } -sub macsuck_filter { \&is_macsuckable_now } -sub macsuck_layer { 2 } - -sub macwalk { (shift)->_walk_body('macsuck', @_) } -sub macsuck { (shift)->_single_body('macsuck', @_) } - -1; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm b/lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm deleted file mode 100644 index 72c7c9ef..00000000 --- a/lib/App/Netdisco/Backend/Worker/Poller/Nbtstat.pm +++ /dev/null @@ -1,73 +0,0 @@ -package App::Netdisco::Backend::Worker::Poller::Nbtstat; - -use Dancer qw/:moose :syntax :script/; -use Dancer::Plugin::DBIC 'schema'; - -use App::Netdisco::Core::Nbtstat qw/nbtstat_resolve_async store_nbt/; -use App::Netdisco::Util::Node 'is_nbtstatable'; -use App::Netdisco::Util::Device qw/get_device is_macsuckable/; -use App::Netdisco::Backend::Util ':all'; - -use NetAddr::IP::Lite ':lower'; -use Time::HiRes 'gettimeofday'; - -use Role::Tiny; -use namespace::clean; - -with 'App::Netdisco::Backend::Worker::Poller::Common'; - -sub nbtstat_action { \&do_nbtstat } -sub nbtstat_filter { \&is_nbtstatable } -sub nbtstat_layer { 2 } - -sub nbtwalk { (shift)->_walk_body('nbtstat', @_) } - -sub nbtstat { - my ($self, $job) = @_; - - my $device = get_device($job->device) - or job_error("nbtstat failed: unable to interpret device parameter"); - my $host = $device->ip; - - unless (is_macsuckable($device)) { - return job_defer("nbtstat deferred: $host is not macsuckable"); - } - - # get list of nodes on device - my $interval = (setting('nbtstat_max_age') || 7) . ' day'; - my $rs = schema('netdisco')->resultset('NodeIp')->search({ - -bool => 'me.active', - -bool => 'nodes.active', - 'nodes.switch' => $device->ip, - 'me.time_last' => \[ '>= now() - ?::interval', $interval ], - },{ - join => 'nodes', - columns => 'ip', - distinct => 1, - })->ip_version(4); - - my @nodes = $rs->get_column('ip')->all; - - # Unless we have IP's don't bother - if (scalar @nodes) { - # filter exclusions from config - @nodes = grep { is_nbtstatable( $_ ) } @nodes; - - # setup the hash nbtstat_resolve_async expects - my @ips = map {+{'ip' => $_}} @nodes; - my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; - - my $resolved_nodes = nbtstat_resolve_async(\@ips); - - # update node_nbt with status entries - foreach my $result (@$resolved_nodes) { - if (defined $result->{'nbname'}) { - store_nbt($result, $now); - } - } - } - - return job_done("Ended nbtstat for $host"); -} - -1; diff --git a/lib/App/Netdisco/Configuration.pm b/lib/App/Netdisco/Configuration.pm index 410b993b..3805c30e 100644 --- a/lib/App/Netdisco/Configuration.pm +++ b/lib/App/Netdisco/Configuration.pm @@ -1,6 +1,7 @@ package App::Netdisco::Configuration; use App::Netdisco::Environment; +use App::Netdisco::Util::SNMP (); use Dancer ':script'; use Path::Class 'dir'; @@ -59,6 +60,9 @@ if ((setting('snmp_auth') and 0 == scalar @{ setting('snmp_auth') }) config->{'community_rw'} = [ @{setting('community_rw')}, 'private' ]; } +# fix up device_auth (or create it from old snmp_auth and community settings) +config->{'device_auth'} = [ App::Netdisco::Util::SNMP::fixup_device_auth() ]; + # defaults for workers setting('workers')->{queue} ||= 'PostgreSQL'; if (exists setting('workers')->{interactives} diff --git a/lib/App/Netdisco/Core/Discover.pm b/lib/App/Netdisco/Core/Discover.pm deleted file mode 100644 index be7c7bcd..00000000 --- a/lib/App/Netdisco/Core/Discover.pm +++ /dev/null @@ -1,996 +0,0 @@ -package App::Netdisco::Core::Discover; - -use Dancer qw/:syntax :script/; -use Dancer::Plugin::DBIC 'schema'; - -use App::Netdisco::Util::Device - qw/get_device match_devicetype is_discoverable/; -use App::Netdisco::Util::Permission qw/check_acl_only check_acl_no/; -use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; -use App::Netdisco::Util::DNS ':all'; -use App::Netdisco::JobQueue qw/jq_queued jq_insert/; -use NetAddr::IP::Lite ':lower'; -use List::MoreUtils (); -use Scalar::Util 'blessed'; -use Encode; -use Try::Tiny; -use NetAddr::MAC; - -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ - set_canonical_ip - store_device store_interfaces store_wireless - store_vlans store_power store_modules - store_neighbors discover_new_neighbors -/; -our %EXPORT_TAGS = (all => \@EXPORT_OK); - -=head1 NAME - -App::Netdisco::Core::Discover - -=head1 DESCRIPTION - -A set of helper subroutines to support parts of the Netdisco application. - -There are no default exports, however the C<:all> tag will export all -subroutines. - -=head1 EXPORT_OK - -=head2 set_canonical_ip( $device, $snmp ) - -Returns: C<$device> - -Given a Device database object, and a working SNMP connection, check whether -the database object's IP is the best choice for that device. If not, update -the IP and hostname in the device object for the canonical IP. - -=cut - -sub set_canonical_ip { - my ($device, $snmp) = @_; - - my $old_ip = $device->ip; - my $new_ip = $old_ip; - my $revofname = ipv4_from_hostname($snmp->name); - - if (setting('reverse_sysname') and $revofname) { - if ($snmp->snmp_connect_ip( $new_ip )) { - $new_ip = $revofname; - } - else { - debug sprintf ' [%s] device - cannot renumber to %s - SNMP connect failed', - $old_ip, $revofname; - } - } - - if (scalar @{ setting('device_identity') }) { - my @idmaps = @{ setting('device_identity') }; - my $devips = $device->device_ips->order_by('alias'); - - ALIAS: while (my $alias = $devips->next) { - next if $alias->alias eq $old_ip; - - foreach my $map (@idmaps) { - next unless ref {} eq ref $map; - - foreach my $key (sort keys %$map) { - # lhs matches device, rhs matches device_ip - if (check_acl_only($device, $key) - and check_acl_only($alias, $map->{$key})) { - - if ($snmp->snmp_connect_ip( $alias->alias )) { - $new_ip = $alias->alias; - last ALIAS; - } - else { - debug sprintf ' [%s] device - cannot renumber to %s - SNMP connect failed', - $old_ip, $alias->alias; - } - } - } - } - } # ALIAS - } - - return if $new_ip eq $old_ip; - - schema('netdisco')->txn_do(sub { - # delete target device with the same vendor and serial number - schema('netdisco')->resultset('Device')->search({ - ip => $new_ip, vendor => $device->vendor, serial => $device->serial, - })->delete; - - # if target device exists then this will die - $device->renumber($new_ip) - or die "cannot renumber to: $new_ip"; # rollback - - debug sprintf ' [%s] device - changed IP to %s (%s)', - $old_ip, $device->ip, ($device->dns || ''); - }); -} - -=head2 store_device( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store basic device information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_device { - my ($device, $snmp) = @_; - - my $ip_index = $snmp->ip_index; - my $interfaces = $snmp->interfaces; - my $ip_netmask = $snmp->ip_netmask; - - # build device aliases suitable for DBIC - my @aliases; - foreach my $entry (keys %$ip_index) { - my $ip = NetAddr::IP::Lite->new($entry) - or next; - my $addr = $ip->addr; - - next if $addr eq '0.0.0.0'; - next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__'); - next if setting('ignore_private_nets') and $ip->is_rfc1918; - - my $iid = $ip_index->{$addr}; - my $port = $interfaces->{$iid}; - my $subnet = $ip_netmask->{$addr} - ? NetAddr::IP::Lite->new($addr, $ip_netmask->{$addr})->network->cidr - : undef; - - debug sprintf ' [%s] device - aliased as %s', $device->ip, $addr; - push @aliases, { - alias => $addr, - port => $port, - subnet => $subnet, - dns => undef, - }; - } - - debug sprintf ' resolving %d aliases with max %d outstanding requests', - scalar @aliases, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; - my $resolved_aliases = hostnames_resolve_async(\@aliases); - - # fake one aliases entry for devices not providing ip_index - push @$resolved_aliases, { alias => $device->ip, dns => $device->dns } - if 0 == scalar @aliases; - - # VTP Management Domain -- assume only one. - my $vtpdomains = $snmp->vtp_d_name; - my $vtpdomain; - if (defined $vtpdomains and scalar values %$vtpdomains) { - $device->set_column( vtp_domain => (values %$vtpdomains)[-1] ); - } - - my $hostname = hostname_from_ip($device->ip); - $device->set_column( dns => $hostname ) if $hostname; - - my @properties = qw/ - snmp_ver - description uptime name - layers ports mac - ps1_type ps2_type ps1_status ps2_status - fan slots - vendor os os_ver - /; - - foreach my $property (@properties) { - $device->set_column( $property => $snmp->$property ); - } - - $device->set_column( model => Encode::decode('UTF-8', $snmp->model) ); - $device->set_column( serial => Encode::decode('UTF-8', $snmp->serial) ); - $device->set_column( contact => Encode::decode('UTF-8', $snmp->contact) ); - $device->set_column( location => Encode::decode('UTF-8', $snmp->location) ); - - - $device->set_column( snmp_class => $snmp->class ); - $device->set_column( last_discover => \'now()' ); - - schema('netdisco')->txn_do(sub { - my $gone = $device->device_ips->delete; - debug sprintf ' [%s] device - removed %d aliases', - $device->ip, $gone; - $device->update_or_insert(undef, {for => 'update'}); - $device->device_ips->populate($resolved_aliases); - debug sprintf ' [%s] device - added %d new aliases', - $device->ip, scalar @aliases; - }); -} - -=head2 store_interfaces( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's interface/port information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_interfaces { - my ($device, $snmp) = @_; - - my $interfaces = $snmp->interfaces; - my $i_type = $snmp->i_type; - my $i_ignore = $snmp->i_ignore; - my $i_descr = $snmp->i_description; - my $i_mtu = $snmp->i_mtu; - my $i_speed = $snmp->i_speed; - my $i_mac = $snmp->i_mac; - my $i_up = $snmp->i_up; - my $i_up_admin = $snmp->i_up_admin; - my $i_name = $snmp->i_name; - my $i_duplex = $snmp->i_duplex; - my $i_duplex_admin = $snmp->i_duplex_admin; - my $i_stp_state = $snmp->i_stp_state; - my $i_vlan = $snmp->i_vlan; - my $i_lastchange = $snmp->i_lastchange; - my $agg_ports = $snmp->agg_ports; - - # clear the cached uptime and get a new one - my $dev_uptime = $snmp->load_uptime; - if (!defined $dev_uptime) { - error sprintf ' [%s] interfaces - Error! Failed to get uptime from device!', - $device->ip; - return; - } - - # used to track how many times the device uptime wrapped - my $dev_uptime_wrapped = 0; - - # use SNMP-FRAMEWORK-MIB::snmpEngineTime if available to - # fix device uptime if wrapped - if (defined $snmp->snmpEngineTime) { - $dev_uptime_wrapped = int( $snmp->snmpEngineTime * 100 / 2**32 ); - if ($dev_uptime_wrapped > 0) { - info sprintf ' [%s] interface - device uptime wrapped %d times - correcting', - $device->ip, $dev_uptime_wrapped; - $device->uptime( $dev_uptime + $dev_uptime_wrapped * 2**32 ); - } - } - - # build device interfaces suitable for DBIC - my %interfaces; - foreach my $entry (keys %$interfaces) { - my $port = $interfaces->{$entry}; - - if (not $port) { - debug sprintf ' [%s] interfaces - ignoring %s (no port mapping)', - $device->ip, $entry; - next; - } - - if (scalar grep {$port =~ m/^$_$/} @{setting('ignore_interfaces') || []}) { - debug sprintf - ' [%s] interfaces - ignoring %s (%s) (config:ignore_interfaces)', - $device->ip, $entry, $port; - next; - } - - if (exists $i_ignore->{$entry}) { - debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s)', - $device->ip, $entry, $port, $i_type->{$entry}; - next; - } - - my $lc = $i_lastchange->{$entry} || 0; - if (not $dev_uptime_wrapped and $lc > $dev_uptime) { - info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting', - $device->ip, $port; - $device->uptime( $dev_uptime + 2**32 ); - $dev_uptime_wrapped = 1; - } - - if ($device->is_column_changed('uptime') and $lc) { - if ($lc < $dev_uptime) { - # ambiguous: lastchange could be sysUptime before or after wrap - if ($dev_uptime > 30000 and $lc < 30000) { - # uptime wrap more than 5min ago but lastchange within 5min - # assume lastchange was directly after boot -> no action - } - else { - # uptime wrap less than 5min ago or lastchange > 5min ago - # to be on safe side, assume lastchange after counter wrap - debug sprintf - ' [%s] interfaces - correcting LastChange for %s, assuming sysUptime wrap', - $device->ip, $port; - $lc += $dev_uptime_wrapped * 2**32; - } - } - } - - $interfaces{$port} = { - port => $port, - descr => $i_descr->{$entry}, - up => $i_up->{$entry}, - up_admin => $i_up_admin->{$entry}, - mac => $i_mac->{$entry}, - speed => $i_speed->{$entry}, - mtu => $i_mtu->{$entry}, - name => Encode::decode('UTF-8', $i_name->{$entry}), - duplex => $i_duplex->{$entry}, - duplex_admin => $i_duplex_admin->{$entry}, - stp => $i_stp_state->{$entry}, - type => $i_type->{$entry}, - vlan => $i_vlan->{$entry}, - pvid => $i_vlan->{$entry}, - is_master => 'false', - slave_of => undef, - lastchange => $lc, - }; - } - - # must do this after building %interfaces so that we can set is_master - foreach my $sidx (keys %$agg_ports) { - my $slave = $interfaces->{$sidx} or next; - my $master = $interfaces->{ $agg_ports->{$sidx} } or next; - next unless exists $interfaces{$slave} and exists $interfaces{$master}; - - $interfaces{$slave}->{slave_of} = $master; - $interfaces{$master}->{is_master} = 'true'; - } - - schema('netdisco')->resultset('DevicePort')->txn_do_locked(sub { - my $gone = $device->ports->delete({keep_nodes => 1}); - debug sprintf ' [%s] interfaces - removed %d interfaces', - $device->ip, $gone; - $device->update_or_insert(undef, {for => 'update'}); - $device->ports->populate([values %interfaces]); - debug sprintf ' [%s] interfaces - added %d new interfaces', - $device->ip, scalar values %interfaces; - }); -} - -=head2 store_wireless( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's wireless interface information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_wireless { - my ($device, $snmp) = @_; - - my $ssidlist = $snmp->i_ssidlist; - return unless scalar keys %$ssidlist; - - my $interfaces = $snmp->interfaces; - my $ssidbcast = $snmp->i_ssidbcast; - my $ssidmac = $snmp->i_ssidmac; - my $channel = $snmp->i_80211channel; - my $power = $snmp->dot11_cur_tx_pwr_mw; - - # build device ssid list suitable for DBIC - my @ssids; - foreach my $entry (keys %$ssidlist) { - (my $iid = $entry) =~ s/\.\d+$//; - my $port = $interfaces->{$iid}; - - if (not $port) { - debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', - $device->ip, $iid; - next; - } - - push @ssids, { - port => $port, - ssid => $ssidlist->{$entry}, - broadcast => $ssidbcast->{$entry}, - bssid => $ssidmac->{$entry}, - }; - } - - schema('netdisco')->txn_do(sub { - my $gone = $device->ssids->delete; - debug sprintf ' [%s] wireless - removed %d SSIDs', - $device->ip, $gone; - $device->ssids->populate(\@ssids); - debug sprintf ' [%s] wireless - added %d new SSIDs', - $device->ip, scalar @ssids; - }); - - # build device channel list suitable for DBIC - my @channels; - foreach my $entry (keys %$channel) { - my $port = $interfaces->{$entry}; - - if (not $port) { - debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', - $device->ip, $entry; - next; - } - - push @channels, { - port => $port, - channel => $channel->{$entry}, - power => $power->{$entry}, - }; - } - - schema('netdisco')->txn_do(sub { - my $gone = $device->wireless_ports->delete; - debug sprintf ' [%s] wireless - removed %d wireless channels', - $device->ip, $gone; - $device->wireless_ports->populate(\@channels); - debug sprintf ' [%s] wireless - added %d new wireless channels', - $device->ip, scalar @channels; - }); -} - -=head2 store_vlans( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's vlan information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_vlans { - my ($device, $snmp) = @_; - - my $v_name = $snmp->v_name; - my $v_index = $snmp->v_index; - - # build device vlans suitable for DBIC - my %v_seen = (); - my @devicevlans; - foreach my $entry (keys %$v_name) { - my $vlan = $v_index->{$entry}; - next unless defined $vlan and $vlan; - ++$v_seen{$vlan}; - - push @devicevlans, { - vlan => $vlan, - description => $v_name->{$entry}, - last_discover => \'now()', - }; - } - - my $i_vlan = $snmp->i_vlan; - my $i_vlan_membership = $snmp->i_vlan_membership; - my $i_vlan_type = $snmp->i_vlan_type; - my $interfaces = $snmp->interfaces; - - # build device port vlans suitable for DBIC - my @portvlans = (); - foreach my $entry (keys %$i_vlan_membership) { - my %port_vseen = (); - my $port = $interfaces->{$entry}; - next unless defined $port; - - my $type = $i_vlan_type->{$entry}; - - foreach my $vlan (@{ $i_vlan_membership->{$entry} }) { - next unless defined $vlan and $vlan; - next if ++$port_vseen{$vlan} > 1; - - my $native = ((defined $i_vlan->{$entry}) and ($vlan eq $i_vlan->{$entry})) ? "t" : "f"; - push @portvlans, { - port => $port, - vlan => $vlan, - native => $native, - vlantype => $type, - last_discover => \'now()', - }; - - next if $v_seen{$vlan}; - - # also add an unnamed vlan to the device - push @devicevlans, { - vlan => $vlan, - description => (sprintf "VLAN %d", $vlan), - last_discover => \'now()', - }; - ++$v_seen{$vlan}; - } - } - - schema('netdisco')->txn_do(sub { - my $gone = $device->vlans->delete; - debug sprintf ' [%s] vlans - removed %d device VLANs', - $device->ip, $gone; - $device->vlans->populate(\@devicevlans); - debug sprintf ' [%s] vlans - added %d new device VLANs', - $device->ip, scalar @devicevlans; - }); - - schema('netdisco')->txn_do(sub { - my $gone = $device->port_vlans->delete; - debug sprintf ' [%s] vlans - removed %d port VLANs', - $device->ip, $gone; - $device->port_vlans->populate(\@portvlans); - debug sprintf ' [%s] vlans - added %d new port VLANs', - $device->ip, scalar @portvlans; - }); -} - -=head2 store_power( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's PoE information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_power { - my ($device, $snmp) = @_; - - my $p_watts = $snmp->peth_power_watts; - my $p_status = $snmp->peth_power_status; - - if (!defined $p_watts) { - debug sprintf ' [%s] power - 0 power modules', $device->ip; - return; - } - - # build device module power info suitable for DBIC - my @devicepower; - foreach my $entry (keys %$p_watts) { - push @devicepower, { - module => $entry, - power => $p_watts->{$entry}, - status => $p_status->{$entry}, - }; - } - - my $interfaces = $snmp->interfaces; - my $p_ifindex = $snmp->peth_port_ifindex; - my $p_admin = $snmp->peth_port_admin; - my $p_pstatus = $snmp->peth_port_status; - my $p_class = $snmp->peth_port_class; - my $p_power = $snmp->peth_port_power; - - # build device port power info suitable for DBIC - my @portpower; - foreach my $entry (keys %$p_ifindex) { - my $port = $interfaces->{ $p_ifindex->{$entry} }; - next unless $port; - - my ($module) = split m/\./, $entry; - - push @portpower, { - port => $port, - module => $module, - admin => $p_admin->{$entry}, - status => $p_pstatus->{$entry}, - class => $p_class->{$entry}, - power => $p_power->{$entry}, - - }; - } - - schema('netdisco')->txn_do(sub { - my $gone = $device->power_modules->delete; - debug sprintf ' [%s] power - removed %d power modules', - $device->ip, $gone; - $device->power_modules->populate(\@devicepower); - debug sprintf ' [%s] power - added %d new power modules', - $device->ip, scalar @devicepower; - }); - - schema('netdisco')->txn_do(sub { - my $gone = $device->powered_ports->delete; - debug sprintf ' [%s] power - removed %d PoE capable ports', - $device->ip, $gone; - $device->powered_ports->populate(\@portpower); - debug sprintf ' [%s] power - added %d new PoE capable ports', - $device->ip, scalar @portpower; - }); -} - -=head2 store_modules( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's module information. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -=cut - -sub store_modules { - my ($device, $snmp) = @_; - - my $e_index = $snmp->e_index; - - if (!defined $e_index) { - schema('netdisco')->txn_do(sub { - my $gone = $device->modules->delete; - debug sprintf ' [%s] modules - removed %d chassis modules', - $device->ip, $gone; - - $device->modules->update_or_create({ - ip => $device->ip, - index => 1, - parent => 0, - name => 'chassis', - class => 'chassis', - pos => -1, - # too verbose and link doesn't work anyway - # description => $device->description, - sw_ver => $device->os_ver, - serial => $device->serial, - model => $device->model, - fru => \'false', - last_discover => \'now()', - }); - }); - - debug - sprintf ' [%s] modules - 0 chassis components (added one pseudo for chassis)', - $device->ip; - - return; - } - - my $e_descr = $snmp->e_descr; - my $e_type = $snmp->e_type; - my $e_parent = $snmp->e_parent; - my $e_name = $snmp->e_name; - my $e_class = $snmp->e_class; - my $e_pos = $snmp->e_pos; - my $e_hwver = $snmp->e_hwver; - my $e_fwver = $snmp->e_fwver; - my $e_swver = $snmp->e_swver; - my $e_model = $snmp->e_model; - my $e_serial = $snmp->e_serial; - my $e_fru = $snmp->e_fru; - - # build device modules list for DBIC - my @modules; - foreach my $entry (keys %$e_index) { - push @modules, { - index => $e_index->{$entry}, - type => $e_type->{$entry}, - parent => $e_parent->{$entry}, - name => Encode::decode('UTF-8', $e_name->{$entry}), - class => $e_class->{$entry}, - pos => $e_pos->{$entry}, - hw_ver => Encode::decode('UTF-8', $e_hwver->{$entry}), - fw_ver => Encode::decode('UTF-8', $e_fwver->{$entry}), - sw_ver => Encode::decode('UTF-8', $e_swver->{$entry}), - model => Encode::decode('UTF-8', $e_model->{$entry}), - serial => Encode::decode('UTF-8', $e_serial->{$entry}), - fru => $e_fru->{$entry}, - description => Encode::decode('UTF-8', $e_descr->{$entry}), - last_discover => \'now()', - }; - } - - schema('netdisco')->txn_do(sub { - my $gone = $device->modules->delete; - debug sprintf ' [%s] modules - removed %d chassis modules', - $device->ip, $gone; - $device->modules->populate(\@modules); - debug sprintf ' [%s] modules - added %d new chassis modules', - $device->ip, scalar @modules; - }); -} - -=head2 store_neighbors( $device, $snmp ) - -returns: C<@to_discover> - -Given a Device database object, and a working SNMP connection, discover and -store the device's port neighbors information. - -Entries in the Topology database table will override any discovered device -port relationships. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -A list of discovererd neighbors will be returned as [C<$ip>, C<$type>] tuples. - -=cut - -sub store_neighbors { - my ($device, $snmp) = @_; - my @to_discover = (); - - # first allow any manually configured topology to be set - _set_manual_topology($device, $snmp); - - if (!defined $snmp->has_topo) { - debug sprintf ' [%s] neigh - neighbor protocols are not enabled', $device->ip; - return @to_discover; - } - - my $interfaces = $snmp->interfaces; - my $c_if = $snmp->c_if; - my $c_port = $snmp->c_port; - my $c_id = $snmp->c_id; - my $c_platform = $snmp->c_platform; - my $c_cap = $snmp->c_cap; - - # v4 and v6 neighbor tables - my $c_ip = ($snmp->c_ip || {}); - my %c_ipv6 = %{ ($snmp->can('hasLLDP') and $snmp->hasLLDP) - ? ($snmp->lldp_ipv6 || {}) : {} }; - - # remove keys with undef values, as c_ip does - delete @c_ipv6{ grep { not defined $c_ipv6{$_} } keys %c_ipv6 }; - # now combine them, v6 wins - $c_ip = { %$c_ip, %c_ipv6 }; - - foreach my $entry (sort (List::MoreUtils::uniq( (keys %$c_ip), (keys %$c_cap) ))) { - if (!defined $c_if->{$entry} or !defined $interfaces->{ $c_if->{$entry} }) { - debug sprintf ' [%s] neigh - port for IID:%s not resolved, skipping', - $device->ip, $entry; - next; - } - - my $port = $interfaces->{ $c_if->{$entry} }; - my $portrow = schema('netdisco')->resultset('DevicePort') - ->single({ip => $device->ip, port => $port}); - - if (!defined $portrow) { - info sprintf ' [%s] neigh - local port %s not in database!', - $device->ip, $port; - next; - } - - if (ref $c_ip->{$entry}) { - error sprintf ' [%s] neigh - Error! port %s has multiple neighbors - skipping', - $device->ip, $port; - next; - } - - my $remote_ip = $c_ip->{$entry}; - my $remote_port = undef; - my $remote_type = Encode::decode('UTF-8', $c_platform->{$entry} || ''); - my $remote_id = Encode::decode('UTF-8', $c_id->{$entry}); - my $remote_cap = $c_cap->{$entry} || []; - - # IP Phone and WAP detection type fixup - if (scalar @$remote_cap or $remote_type) { - my $phone_flag = grep {match_devicetype($_, 'phone_capabilities')} - @$remote_cap; - my $ap_flag = grep {match_devicetype($_, 'wap_capabilities')} - @$remote_cap; - - if ($phone_flag or match_devicetype($remote_type, 'phone_platforms')) { - $remote_type = 'IP Phone: '. $remote_type - if $remote_type !~ /ip.phone/i; - } - elsif ($ap_flag or match_devicetype($remote_type, 'wap_platforms')) { - $remote_type = 'AP: '. $remote_type; - } - - $portrow->update({remote_type => $remote_type}); - } - - if ($portrow->manual_topo) { - info sprintf ' [%s] neigh - %s has manually defined topology', - $device->ip, $port; - next; - } - - next unless $remote_ip; - - # a bunch of heuristics to search known devices if we don't have a - # useable remote IP... - - if ($remote_ip eq '0.0.0.0' or - check_acl_no($remote_ip, 'group:__LOCAL_ADDRESSES__')) { - - if ($remote_id) { - my $devices = schema('netdisco')->resultset('Device'); - my $neigh = $devices->single({name => $remote_id}); - info sprintf - ' [%s] neigh - bad address %s on port %s, searching for %s instead', - $device->ip, $remote_ip, $port, $remote_id; - - if (!defined $neigh) { - my $mac = NetAddr::MAC->new(mac => $remote_id); - if ($mac and not $mac->errstr) { - $neigh = $devices->single({mac => $mac->as_ieee}); - } - } - - # some HP switches send 127.0.0.1 as remote_ip if no ip address - # on default vlan for HP switches remote_ip looks like - # "myswitchname(012345-012345)" - if (!defined $neigh) { - (my $tmpid = $remote_id) =~ s/.*\(([0-9a-f]{6})-([0-9a-f]{6})\).*/$1$2/; - my $mac = NetAddr::MAC->new(mac => $tmpid); - if ($mac and not $mac->errstr) { - info sprintf - '[%s] neigh - found neighbor %s by MAC %s', - $device->ip, $remote_id, $mac->as_ieee; - $neigh = $devices->single({mac => $mac->as_ieee}); - } - } - - if (!defined $neigh) { - (my $shortid = $remote_id) =~ s/\..*//; - $neigh = $devices->single({name => { -ilike => "${shortid}%" }}); - } - - if ($neigh) { - $remote_ip = $neigh->ip; - info sprintf ' [%s] neigh - found %s with IP %s', - $device->ip, $remote_id, $remote_ip; - } - else { - info sprintf ' [%s] neigh - could not find %s, skipping', - $device->ip, $remote_id; - next; - } - } - else { - info sprintf ' [%s] neigh - skipping unuseable address %s on port %s', - $device->ip, $remote_ip, $port; - next; - } - } - - # what we came here to do.... discover the neighbor - debug sprintf - ' [%s] neigh - adding neighbor %s, type [%s], on %s to discovery queue', - $device->ip, $remote_ip, ($remote_type || ''), $port; - push @to_discover, [$remote_ip, $remote_type]; - - $remote_port = $c_port->{$entry}; - if (defined $remote_port) { - # clean weird characters - $remote_port =~ s/[^\d\/\.,()\w:-]+//gi; - } - else { - info sprintf ' [%s] neigh - no remote port found for port %s at %s', - $device->ip, $port, $remote_ip; - } - - $portrow->update({ - remote_ip => $remote_ip, - remote_port => $remote_port, - remote_type => $remote_type, - remote_id => $remote_id, - is_uplink => \"true", - manual_topo => \"false", - }); - - # update master of our aggregate to be a neighbor of - # the master on our peer device (a lot of iffs to get there...). - # & cannot use ->neighbor prefetch because this is the port insert! - if (defined $portrow->slave_of) { - - my $peer_device = get_device($remote_ip); - my $master = schema('netdisco')->resultset('DevicePort')->single({ - ip => $device->ip, - port => $portrow->slave_of - }); - - if ($peer_device and $peer_device->in_storage and $master - and not ($portrow->is_master or defined $master->slave_of)) { - - my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ - ip => $peer_device->ip, - port => $portrow->remote_port, - }); - - $master->update({ - remote_ip => ($peer_device->ip || $remote_ip), - remote_port => ($peer_port ? $peer_port->slave_of : undef ), - is_uplink => \"true", - is_master => \"true", - manual_topo => \"false", - }); - } - } - } - - return @to_discover; -} - -# take data from the topology table and update remote_ip and remote_port -# in the devices table. only use root_ips and skip any bad topo entries. -sub _set_manual_topology { - my ($device, $snmp) = @_; - - schema('netdisco')->txn_do(sub { - # clear manual topology flags - schema('netdisco')->resultset('DevicePort') - ->search({ip => $device->ip})->update({manual_topo => \'false'}); - - my $topo_links = schema('netdisco')->resultset('Topology') - ->search({-or => [dev1 => $device->ip, dev2 => $device->ip]}); - debug sprintf ' [%s] neigh - setting manual topology links', $device->ip; - - while (my $link = $topo_links->next) { - # could fail for broken topo, but we ignore to try the rest - try { - schema('netdisco')->txn_do(sub { - # only work on root_ips - my $left = get_device($link->dev1); - my $right = get_device($link->dev2); - - # skip bad entries - return unless ($left->in_storage and $right->in_storage); - - $left->ports - ->single({port => $link->port1}) - ->update({ - remote_ip => $right->ip, - remote_port => $link->port2, - remote_type => undef, - remote_id => undef, - is_uplink => \"true", - manual_topo => \"true", - }); - - $right->ports - ->single({port => $link->port2}) - ->update({ - remote_ip => $left->ip, - remote_port => $link->port1, - remote_type => undef, - remote_id => undef, - is_uplink => \"true", - manual_topo => \"true", - }); - }); - }; - } - }); -} - -=head2 discover_new_neighbors( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, discover and -store the device's port neighbors information. - -Entries in the Topology database table will override any discovered device -port relationships. - -The Device database object can be a fresh L object which is -not yet stored to the database. - -Any discovered neighbor unknown to Netdisco will have a C job -immediately queued (subject to the filtering by the C settings). - -=cut - -sub discover_new_neighbors { - my @to_discover = store_neighbors(@_); - - # only enqueue if device is not already discovered, - # discover_* config permits the discovery - foreach my $neighbor (@to_discover) { - my ($ip, $remote_type) = @$neighbor; - - my $device = get_device($ip); - next if $device->in_storage; - - if (not is_discoverable($device, $remote_type)) { - debug sprintf - ' queue - %s, type [%s] excluded by discover_* config', - $ip, ($remote_type || ''); - next; - } - - jq_insert({ - device => $ip, - action => 'discover', - subaction => 'with-nodes', - }); - } -} - -1; diff --git a/lib/App/Netdisco/DB/Result/Device.pm b/lib/App/Netdisco/DB/Result/Device.pm index 820d77b1..6352e444 100644 --- a/lib/App/Netdisco/DB/Result/Device.pm +++ b/lib/App/Netdisco/DB/Result/Device.pm @@ -10,6 +10,8 @@ use warnings; use NetAddr::IP::Lite ':lower'; use App::Netdisco::Util::DNS 'hostname_from_ip'; +use overload '""' => sub { shift->ip }, fallback => 1; + use base 'DBIx::Class::Core'; __PACKAGE__->table("device"); __PACKAGE__->add_columns( @@ -191,6 +193,29 @@ __PACKAGE__->might_have( =head1 ADDITIONAL METHODS +=head2 is_pseudo + +Returns true if the vendor of the device is "netdisco". + +=cut + +sub is_pseudo { + my $device = shift; + return (defined $device->vendor and $device->vendor eq 'netdisco'); +} + +=head2 has_layer( $number ) + +Returns true if the device provided sysServices and supports the given layer. + +=cut + +sub has_layer { + my ($device, $layer) = @_; + return unless $layer and $layer =~ m/^[1-7]$/; + return ($device->layers and (substr($device->layers, (8-$layer), 1) == 1)); +} + =head2 renumber( $new_ip ) Will update this device and all related database records to use the new IP diff --git a/lib/App/Netdisco/JobQueue/PostgreSQL.pm b/lib/App/Netdisco/JobQueue/PostgreSQL.pm index 0f1eda9c..d7b73d53 100644 --- a/lib/App/Netdisco/JobQueue/PostgreSQL.pm +++ b/lib/App/Netdisco/JobQueue/PostgreSQL.pm @@ -153,12 +153,12 @@ sub jq_lock { try { schema('netdisco')->txn_do(sub { schema('netdisco')->resultset('Admin') - ->search({ job => $job->job }, { for => 'update' }) + ->search({ job => $job->id }, { for => 'update' }) ->update({ status => ('queued-'. setting('workers')->{'BACKEND'}) }); return unless schema('netdisco')->resultset('Admin') - ->count({ job => $job->job, + ->count({ job => $job->id, status => ('queued-'. setting('workers')->{'BACKEND'}) }); # remove any duplicate jobs, needed because we have race conditions @@ -202,7 +202,7 @@ sub jq_defer { # lock db row and update to show job is available schema('netdisco')->resultset('Admin') - ->find($job->job, {for => 'update'}) + ->find($job->id, {for => 'update'}) ->update({ status => 'queued', started => undef }); }); $happy = true; @@ -233,7 +233,7 @@ sub jq_complete { } schema('netdisco')->resultset('Admin') - ->find($job->job, {for => 'update'})->update({ + ->find($job->id, {for => 'update'})->update({ status => $job->status, log => $job->log, started => $job->started, diff --git a/lib/App/Netdisco/Manual/Configuration.pod b/lib/App/Netdisco/Manual/Configuration.pod index a563780a..d91a6c82 100644 --- a/lib/App/Netdisco/Manual/Configuration.pod +++ b/lib/App/Netdisco/Manual/Configuration.pod @@ -52,8 +52,11 @@ val2} >> on one line or C on separate lines, e.g.: =item * -String - Quoted, just like in Perl (and essential if the item contains the -colon character). +String - Optionally quoted, just like in Perl (quotes required if the item +contains a colon character). For example: + + log: 'debug' + expire_devices: 15 =back @@ -424,7 +427,7 @@ indenting the query text. =head4 C (optional) Section of the Reports menu where this report will appear. See -L for the full list. +L for the full list. If not supplied, reports appear in a I category. =head4 C (optional) @@ -581,7 +584,7 @@ Value: Dictionary of Access Control Lists. Default: None. Several configuration settings in Netdisco make use of L to identify lists of devices or hosts. Examples are the C<*_no> settings such as C, the C<*_only> settings such as C, -and some "C" settings which appear in C and C +and some "C" settings which appear in C and C configuration. The C setting allows for naming of groups which are then @@ -645,11 +648,11 @@ renumbered to use that interface as its new identity and the process stops. When using an Access Control List for the value (interface selection), as well as the options described in L you can use "C" to match an interface's port name. For example to renumber -all Arista devices to the IP and host name of their Mgmt1 interface (if they -have one), you could use: +any device to the IP and host name of its Mgmt1 interface (if it has one), you +could use: device_identity: - 'vendor:arista': 'port:(?i)mgmt1' + 'any': 'port:(?i)mgmt1' Once a device is renumbered, its new identity is "sticky". That is, you could remove the C configuration and the next "discover" job will @@ -711,7 +714,7 @@ Each is tried in turn when polling the device, and then the working community string will be cached in the database. For fine-grained control over which communities are tried for which devices, -or to set SNMPv3 authentication, see C, below. +or to set SNMPv3 authentication, see C, below. =head3 C @@ -725,13 +728,13 @@ is tried in turn when writing to the device, and then the working community string will be cached in the database. For fine-grained control over which communities are tried for which devices, -or to set SNMPv3 authentication, see C, below. +or to set SNMPv3 authentication, see C, below. -=head3 C +=head3 C Value: List of Settings Trees. Default: Empty List. -This setting configures authenticaiton for all SNMP versions, and provides an +This setting configures authenticaiton for all polling, and provides an alternative fine-grained control for SNMPv1 and SNMPv2 community strings. You provide a list of authentication "I", and Netdisco will try each in turn, then cache the one which works for a device. @@ -741,7 +744,7 @@ limited to read (get) and/or write (set) operations. By default, a stanza is enabled for all device IPs, for read access only. The "tag" of a stanza is simply a friendly name used by Netdisco when referring to the configuration. - snmp_auth: + device_auth: - community: public - community: publictwo - community: mycommunity @@ -757,26 +760,23 @@ simply a friendly name used by Netdisco when referring to the configuration. priv: pass: netdiscokey2 proto: DES - - tag: v3aclexample - user: netdisco2 + - tag: aclexample + community: s3kr1t + read: false + write: true only: - 192.0.2.0/30 - 172.20.10.0/24 no: '172.20.10.1' - - tag: v2aclexample - community: s3kr1t - read: false - write: true - only: '2001:db8::/32' For SNMPv1 and SNMPv2, only the C key is required. Unlike the global C/C setting, this is not a list but a single -item. That is, to configure multiple community strings, have one stanza per +item. Therefore, to configure multiple community strings, have one stanza per community, as in the examples above and below. -For any version of SNMP you can add C and/or C booleans to -control operations for that stanza, and IP restrictions using C and -C (see L for what you can use here). +For any sanza you can add C and/or C booleans to control whether +it is used for get and/or set operations, and IP restrictions using C +and C (see L for what you can use here). For SNMPv3 the C and C keys are required. Providing an C section enables the authentication security level, providing a C section @@ -794,6 +794,26 @@ this you usually configure a common context "prefix", with Netdisco's default being "C" (i.e. C, C, etc). Add the C key to a stanza to override this default. +For other authentication mechanisms (HTTP, SSH, etc), C is also required. +Each transport will have different settings, but usually a C and +C are required, and optionally some other settings. See the +transport or driver documentation pages for further details. For example: + + device_auth: + - tag: ye_olde_snmp + community: public + - tag: sshcollector + only: 'group:sshcollectordevices' + driver: cli + method: arpnip_nodes + username: foo + password: bar + - tag: netconf_devices + only: 'vendor:juniper' + driver: netconf + username: oliver + password: letmein + Netdisco caches both the successful SNMPv2 read and write community strings, as well as the C names if available. This allows for faster operations once a connection has previously been made to a device. Tags are recommended. @@ -806,7 +826,7 @@ Finally, a reminder that multiple SNMPv2 community strings need to be in separate named stanza, as below. However for simple v2 configurations you can revert to the "C" setting, instead: - snmp_auth: + device_auth: - tag: 'default_v2_readonly1' community: 'read1' - tag: 'default_v2_readonly2' @@ -822,7 +842,7 @@ Value: String. Default none. An external program to run to get the community string for a given device. This is useful if, for example, you have you devices already configured in another NMS and you want to use that information instead of configuring -C. +C. The strings "C<%IP%>" and "C<%HOST%>" are replaced by the IP address and the hostname (or IP address if no hostname is known) of the system being @@ -836,7 +856,7 @@ The command must return output in the following form: setCommunity= If the community string is not known for the given system, the command should -return no output and the community strings configured in C, +return no output and the community strings configured in C, C, and C will be used instead. =head3 C diff --git a/lib/App/Netdisco/Manual/Developing.pod b/lib/App/Netdisco/Manual/Developing.pod index b580b281..e581f8b0 100644 --- a/lib/App/Netdisco/Manual/Developing.pod +++ b/lib/App/Netdisco/Manual/Developing.pod @@ -311,7 +311,7 @@ Every Dancer route handler must have proper role based access control enabled, to prevent unauthorized access to Netdisco's data, or admin features. This is done with the L module. It handles both the authentication using Netdisco's database, and then protects each route -handler. See L for details. +handler. See L for details. =head2 Templates diff --git a/lib/App/Netdisco/Manual/WritingPlugins.pod b/lib/App/Netdisco/Manual/WritingWebPlugins.pod similarity index 99% rename from lib/App/Netdisco/Manual/WritingPlugins.pod rename to lib/App/Netdisco/Manual/WritingWebPlugins.pod index c6321633..fd89cb44 100644 --- a/lib/App/Netdisco/Manual/WritingPlugins.pod +++ b/lib/App/Netdisco/Manual/WritingWebPlugins.pod @@ -1,6 +1,6 @@ =head1 NAME -App::Netdisco::Manual::WritingPlugins - Documentation on Plugins for Developers +App::Netdisco::Manual::WritingWebPlugins - Documentation on Web Plugins for Developers =head1 Introduction @@ -26,7 +26,7 @@ App::Netdisco plugins should load the L module. This exports a set of helper subroutines to register the new UI components. Here's the boilerplate code for our example plugin module: - package App::Netdisco::Web::Plugin::MyNewFeature + package App::Netdisco::Web::Plugin::MyNewFeature; use Dancer ':syntax'; use Dancer::Plugin::DBIC; diff --git a/lib/App/Netdisco/Manual/WritingWorkers.pod b/lib/App/Netdisco/Manual/WritingWorkers.pod new file mode 100644 index 00000000..69d49528 --- /dev/null +++ b/lib/App/Netdisco/Manual/WritingWorkers.pod @@ -0,0 +1,226 @@ +=head1 NAME + +App::Netdisco::Manual::WritingWorkers - Developer Documentation on Worker Plugins + +=head1 Introduction + +L's plugin system allows users to write I to gather +information from network devices using different I and store +results in the database. + +For example, transports might be SNMP, SSH, or HTTPS. Workers might be +combining those transports with application protocols such as SNMP, NETCONF +(OpenConfig with XML), RESTCONF (OpenConfig with JSON), eAPI, or even CLI +scraping. The combination of transport and protocol is known as a I. + +Workers can be restricted to certain vendor platforms using familiar ACL +syntax. They are also attached to specific actions in Netdisco's backend +operation (discover, macsuck, etc). + +See L for more information about worker +plugins. + +=head1 Developing Workers + +A worker is Perl code which is run. Therefore it can do anything you like, but +typically it will make a connection to a device, gather some data, and store +it in Netdisco's database. + +App::Netdisco plugins must load the L module. +This exports a helper subroutine to register the worker. Here's the +boilerplate code for our example plugin module: + + package App::Netdisco::Worker::Plugin::Discover::Wireless::UniFi; + + use Dancer ':syntax'; + use App::Netdisco::Worker::Plugin; + use aliased 'App::Netdisco::Worker::Status'; + + # worker registration code goes here, ** see below ** + + true; + +=head1 Registering a Worker + +Use the C helper from L to +register a worker: + + register_worker( $coderef ); + # or + register_worker( \%workerconf, $coderef ); + +For example (using the second form): + + register_worker({ + driver => 'unifiapi', + }, sub { "worker code here" }); + +The C<%workerconf> hashref is optional, and described below. The C<$coderef> +is the main body of your worker. Your worker is run in a L +statement to catch errors, and passed the following arguments: + + $coderef->($job, \%workerconf); + +The C<$job> is an instance of L. Note that this +class has a C slot which may be filled, depending on the action, and +if the device is not yet discovered then the row will not yet be in storage. +The C<\%workerconf> hashref is the set of configuration parameters you used +to declare the worker (documented below). + +=head2 Package Naming Convention + +The package name used where the worker is declared is significant. Let's look +at the boilerplate example again: + + package App::Netdisco::Worker::Plugin::Discover::Wireless::UniFi; + +The package name B contain C and the namespace component after +that becomes the action. For example workers registered in the above package +will be run during the I backend action (that is, during a +C job). You can replace C with other actions such as +C, C, C, and C, or create your own. + +The component after the action is known as the I (C in this +example), and is the way to override a Netdisco built-in worker, by using the +same name (plus an entry in C<%workerconf>, see below). Otherwise you can use +any valid Perl bareword for the phase. + +Workers may also be registered directly to the action (C, in this +example), without any phase. This is used for very early bootstrapping code +(such as first inserting a device into the database so it can be used by +subsequent phases) or for very simple, generic actions (such as C). + +=head2 C<%workerconf> Options + +=over 4 + +=item ACL Options + +Workers may have C and C parameters configured which use the +standard ACL syntax described in L. The C directive is +especially useful as it can restrict a worker to a given device platform or +operating system (for example Cisco IOS XR for the C driver). + +=item C (string) + +The driver is a label associated with a group of workers and typically refers +to the combination of transport and application protocol. Examples include +C, C, C, C, and C. The convention is for +driver names to be lowercase. + +Users will bind authentication configuration settings to drivers in their +configuration. If no driver is specified when registering a worker, it will be +run for every device and phase (such as during Expire jobs). + +=item C (boolean) + +When multiple workers are registered for the same phase, they will all be run. +However there is a special "I" slot for each phase in which only one +worker (the first that succeeds) is used. Most of Netdisco's built-in worker +code is registered in this way, so to override it you can use the same package +namespace and set C to be C. + +=back + +=head1 Worker Execution and Return Code + +Workers are configured as an ordered list. They are grouped by C and +C (as in Package Naming Convention, above). + +Workers defined in C are run before those in +C so you have an opportunity to override built-in workers by +adding them to C and setting C to C in +the worker configuration. + +The return code of the worker is significant for those configured with +C as C: when the worker returns true, no other C hooks +are run for that phase. You should always use the aliased +L helper (loaded as in the boilerplate code +above) when returning a value, such as: + + return Status->done('everything is good'); + # or + return Status->error('something went wrong'); + # or + return Status->defer('this device cannot be processed right now'); + +Remember that a worker is only run if it matches the hardware platform of the +target device and the user's configuration, and is not also excluded by the +user's configuration. This filtering takes place before inspecting C. + +=head1 Accessing Transports + +From your worker you will want to connect to a device to gather data. This is +done using a transport protocol session (SNMP, SSH, etc). Transports are +singleton objects instantiated on demand, so they can be shared among a set of +workers that are accessing the same device. + +See the documentation for each transport to find out how to access it: + +=over 4 + +=item * + +L + +=back + +=head1 Database Connections + +The Netdisco database is available via the C schema key, as below. +You can also use the C configuration item to set up +connections to other databases. + + # plugin package + use Dancer::Plugin::DBIC; + my $set = + schema('netdisco')->resultset('Devices') + ->search({vendor => 'cisco'}); + +=head1 Review of Terminology + +In summary, Worker code is defined in a package namespace specifying the +Action and Phase, and registered as a plugin with configuration which may +specify the Driver and whether it is in the Primary slot. Access Control Lists +determine which Workers are permitted to run, and when. Here are more complete +definitions: + +=over 4 + +=item C + +The highest level grouping of workers, corresponding to a Netdisco command +such as C or C. Workers can be registered at this level to +do really early bootstrapping work. + +=item C + +The next level down from C for grouping workers. Phases have arbitrary +names and are visited in the order defined in the C +setting list, followed by the C setting list. Workers are +usually registered at this level. + +=item C + +A lump of code you write which does a single clearly defined task. The package +namespace of the worker identifies the action and optionally the phase. +Workers are typically registered with some configuration settings. + +=item C + +A label associated with a group of workers which refers to a combination of +transport and application protocol used to connect to and communicate with the +target device. Users attach authentication configuration to specific drivers. + +=item C (defaults to C) + +Indicates that the worker will only be run if no other C worker for +this phase has already succeeded. In this way, you can override Netdisco code +by setting this option and returning true from your worker. + +=back + +=cut + diff --git a/lib/App/Netdisco/Transport/SNMP.pm b/lib/App/Netdisco/Transport/SNMP.pm new file mode 100644 index 00000000..34db7dbd --- /dev/null +++ b/lib/App/Netdisco/Transport/SNMP.pm @@ -0,0 +1,312 @@ +package App::Netdisco::Transport::SNMP; + +use Dancer qw/:syntax :script/; +use Dancer::Plugin::DBIC 'schema'; + +use App::Netdisco::Util::SNMP 'get_communities'; +use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::Permission ':all'; + +use SNMP::Info; +use Try::Tiny; +use Module::Load (); +use Path::Class 'dir'; +use NetAddr::IP::Lite ':lower'; + +use base 'Dancer::Object::Singleton'; + +=head1 NAME + +App::Netdisco::Transport::SNMP + +=head1 DESCRIPTION + +Singleton for SNMP connections. Returns cached L instance for a +given device IP, or else undef. All methods are class methods, for example: + + App::Netdisco::Transport::SNMP->reader_for( ... ); + +=cut + +__PACKAGE__->attributes(qw/ readers writers /); + +sub init { + my ( $class, $self ) = @_; + $self->readers( {} ); + $self->writers( {} ); + return $self; +} + +=head1 reader_for( $ip, $useclass? ) + +Given an IP address, returns an L instance configured for and +connected to that device. The IP can be any on the device, and the management +interface will be connected to. + +If the device is known to Netdisco and there is a cached SNMP community +string, that community will be tried first, and then other community strings +from the application configuration will be tried. + +If C<$useclass> is provided, it will be used as the L device +class instead of the class in the Netdisco database. + +Returns C if the connection fails. + +=cut + +sub reader_for { + my ($class, $ip, $useclass) = @_; + my $device = get_device($ip) or return undef; + my $readers = $class->instance->readers or return undef; + return $readers->{$device->ip} if exists $readers->{$device->ip}; + debug sprintf 'snmp reader cache warm: [%s]', $device->ip; + return ($readers->{$device->ip} + = _snmp_connect_generic('read', $device, $useclass)); +} + +=head1 test_connection( $ip ) + +Similar to C but will use the literal IP address passed, and does +not support specifying the device class. The purpose is to test the SNMP +connectivity to the device before a renumber. + +Attempts to have no side effect, however there will be a stored SNMP +authentication hint (tag) in the database if the connection is successful. + +Returns C if the connection fails. + +=cut + +sub test_connection { + my ($class, $ip) = @_; + my $addr = NetAddr::IP::Lite->new($ip) or return undef; + # avoid renumbering to localhost loopbacks + return undef if $addr->addr eq '0.0.0.0' + or check_acl_no($addr->addr, 'group:__LOCAL_ADDRESSES__'); + my $device = schema('netdisco')->resultset('Device') + ->new_result({ ip => $addr->addr }) or return undef; + my $readers = $class->instance->readers or return undef; + return $readers->{$device->ip} if exists $readers->{$device->ip}; + debug sprintf 'snmp reader cache warm: [%s]', $device->ip; + return ($readers->{$device->ip} = _snmp_connect_generic('read', $device)); +} + +=head1 writer_for( $ip, $useclass? ) + +Same as C but uses the read-write community strings from the +application configuration file. + +Returns C if the connection fails. + +=cut + +sub writer_for { + my ($class, $ip, $useclass) = @_; + my $device = get_device($ip) or return undef; + my $writers = $class->instance->writers or return undef; + return $writers->{$device->ip} if exists $writers->{$device->ip}; + debug sprintf 'snmp writer cache warm: [%s]', $device->ip; + return ($writers->{$device->ip} + = _snmp_connect_generic('write', $device, $useclass)); +} + +sub _snmp_connect_generic { + my ($mode, $device, $useclass) = @_; + $mode ||= 'read'; + + my %snmp_args = ( + AutoSpecify => 0, + DestHost => $device->ip, + # 0 is falsy. Using || with snmpretries equal to 0 will set retries to 2. + # check if the setting is 0. If not, use the default value of 2. + Retries => (setting('snmpretries') || setting('snmpretries') == 0 ? 0 : 2), + Timeout => (setting('snmptimeout') || 1000000), + NonIncreasing => (setting('nonincreasing') || 0), + BulkWalk => ((defined setting('bulkwalk_off') && setting('bulkwalk_off')) + ? 0 : 1), + BulkRepeaters => (setting('bulkwalk_repeaters') || 20), + MibDirs => [ _build_mibdirs() ], + IgnoreNetSNMPConf => 1, + Debug => ($ENV{INFO_TRACE} || 0), + DebugSNMP => ($ENV{SNMP_TRACE} || 0), + ); + + # an override for bulkwalk + $snmp_args{BulkWalk} = 0 if check_acl_no($device, 'bulkwalk_no'); + + # further protect against buggy Net-SNMP, and disable bulkwalk + if ($snmp_args{BulkWalk} + and ($SNMP::VERSION eq '5.0203' || $SNMP::VERSION eq '5.0301')) { + + warning sprintf + "[%s] turning off BulkWalk due to buggy Net-SNMP - please upgrade!", + $device->ip; + $snmp_args{BulkWalk} = 0; + } + + # get the community string(s) + my @communities = get_communities($device, $mode); + + # which SNMP versions to try and in what order + my @versions = + ( check_acl_no($device->ip, 'snmpforce_v3') ? (3) + : check_acl_no($device->ip, 'snmpforce_v2') ? (2) + : check_acl_no($device->ip, 'snmpforce_v1') ? (1) + : (reverse (1 .. (setting('snmpver') || 3))) ); + + # use existing or new device class + my @classes = ($useclass || 'SNMP::Info'); + if ($device->snmp_class and not $useclass) { + unshift @classes, $device->snmp_class; + } + + my $info = undef; + COMMUNITY: foreach my $comm (@communities) { + next unless $comm; + + VERSION: foreach my $ver (@versions) { + next unless $ver; + + next if $ver eq 3 and exists $comm->{community}; + next if $ver ne 3 and !exists $comm->{community}; + + CLASS: foreach my $class (@classes) { + next unless $class; + + my %local_args = (%snmp_args, Version => $ver); + $info = _try_connect($device, $class, $comm, $mode, \%local_args, + ($useclass ? 0 : 1) ); + last COMMUNITY if $info; + } + } + } + + return $info; +} + +sub _try_connect { + my ($device, $class, $comm, $mode, $snmp_args, $reclass) = @_; + my %comm_args = _mk_info_commargs($comm); + my $debug_comm = ''; + if ($ENV{SHOW_COMMUNITY}) { + $debug_comm = ($comm->{community} || + (sprintf 'v3:%s:%s/%s', ($comm->{user}, + ($comm->{auth}->{proto} || 'noAuth'), + ($comm->{priv}->{proto} || 'noPriv'))) ); + } + my $info = undef; + + try { + debug + sprintf '[%s] try_connect with ver: %s, class: %s, comm: %s', + $snmp_args->{DestHost}, $snmp_args->{Version}, $class, $debug_comm; + Module::Load::load $class; + + $info = $class->new(%$snmp_args, %comm_args) or return; + $info = ($mode eq 'read' ? _try_read($info, $device, $comm) + : _try_write($info, $device, $comm)); + + # first time a device is discovered, re-instantiate into specific class + if ($reclass and $info and $info->device_type ne $class) { + $class = $info->device_type; + debug + sprintf '[%s] try_connect with ver: %s, new class: %s, comm: %s', + $snmp_args->{DestHost}, $snmp_args->{Version}, $class, $debug_comm; + + Module::Load::load $class; + $info = $class->new(%$snmp_args, %comm_args); + } + } + catch { + debug $_; + }; + + return $info; +} + +sub _try_read { + my ($info, $device, $comm) = @_; + + return undef unless ( + (not defined $info->error) + and defined $info->uptime + and ($info->layers or $info->description) + and $info->class + ); + + $device->in_storage + ? $device->update({snmp_ver => $info->snmp_ver}) + : $device->set_column(snmp_ver => $info->snmp_ver); + + if ($comm->{community}) { + $device->in_storage + ? $device->update({snmp_comm => $comm->{community}}) + : $device->set_column(snmp_comm => $comm->{community}); + } + + # regardless of device in storage, save the hint + $device->update_or_create_related('community', + {snmp_auth_tag_read => $comm->{tag}}) if $comm->{tag}; + + return $info; +} + +sub _try_write { + my ($info, $device, $comm) = @_; + + my $loc = $info->load_location; + $info->set_location($loc) or return undef; + return undef unless ($loc eq $info->load_location); + + $device->in_storage + ? $device->update({snmp_ver => $info->snmp_ver}) + : $device->set_column(snmp_ver => $info->snmp_ver); + + # one of these two cols must be set + $device->update_or_create_related('community', { + ($comm->{tag} ? (snmp_auth_tag_write => $comm->{tag}) : ()), + ($comm->{community} ? (snmp_comm_rw => $comm->{community}) : ()), + }); + + return $info; +} + +sub _mk_info_commargs { + my $comm = shift; + return () unless ref {} eq ref $comm and scalar keys %$comm; + + return (Community => $comm->{community}) + if exists $comm->{community}; + + my $seclevel = + (exists $comm->{auth} ? + (exists $comm->{priv} ? 'authPriv' : 'authNoPriv' ) + : 'noAuthNoPriv'); + + return ( + SecName => $comm->{user}, + SecLevel => $seclevel, + ( exists $comm->{auth} ? ( + AuthProto => uc ($comm->{auth}->{proto} || 'MD5'), + AuthPass => ($comm->{auth}->{pass} || ''), + ( exists $comm->{priv} ? ( + PrivProto => uc ($comm->{priv}->{proto} || 'DES'), + PrivPass => ($comm->{priv}->{pass} || ''), + ) : ()), + ) : ()), + ); +} + +sub _build_mibdirs { + my $home = (setting('mibhome') || dir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs')); + return map { dir($home, $_)->stringify } + @{ setting('mibdirs') || _get_mibdirs_content($home) }; +} + +sub _get_mibdirs_content { + my $home = shift; + my @list = map {s|$home/||; $_} grep {m/[a-z0-9]/} grep {-d} glob("$home/*"); + return \@list; +} + +true; diff --git a/lib/App/Netdisco/Util/Device.pm b/lib/App/Netdisco/Util/Device.pm index a45c631f..f86d6af2 100644 --- a/lib/App/Netdisco/Util/Device.pm +++ b/lib/App/Netdisco/Util/Device.pm @@ -192,12 +192,11 @@ sub is_discoverable_now { my ($ip, $remote_type) = @_; my $device = get_device($ip) or return 0; - if ($device->in_storage) { - if ($device->since_last_discover and setting('discover_min_age') - and $device->since_last_discover < setting('discover_min_age')) { + if ($device->in_storage + and $device->since_last_discover and setting('discover_min_age') + and $device->since_last_discover < setting('discover_min_age')) { - return _bail_msg("is_discoverable: time since last discover less than discover_min_age"); - } + return _bail_msg("is_discoverable: time since last discover less than discover_min_age"); } return is_discoverable(@_); @@ -242,10 +241,8 @@ sub is_arpnipable_now { my ($ip) = @_; my $device = get_device($ip) or return 0; - return _bail_msg("is_arpnipable: cannot arpnip an undiscovered device") - if not $device->in_storage; - - if ($device->since_last_arpnip and setting('arpnip_min_age') + if ($device->in_storage + and $device->since_last_arpnip and setting('arpnip_min_age') and $device->since_last_arpnip < setting('arpnip_min_age')) { return _bail_msg("is_arpnipable: time since last arpnip less than arpnip_min_age"); @@ -293,10 +290,8 @@ sub is_macsuckable_now { my ($ip) = @_; my $device = get_device($ip) or return 0; - return _bail_msg("is_macsuckable: cannot macsuck an undiscovered device") - if not $device->in_storage; - - if ($device->since_last_macsuck and setting('macsuck_min_age') + if ($device->in_storage + and $device->since_last_macsuck and setting('macsuck_min_age') and $device->since_last_macsuck < setting('macsuck_min_age')) { return _bail_msg("is_macsuckable: time since last macsuck less than macsuck_min_age"); diff --git a/lib/App/Netdisco/Util/Backend.pm b/lib/App/Netdisco/Util/MCE.pm similarity index 91% rename from lib/App/Netdisco/Util/Backend.pm rename to lib/App/Netdisco/Util/MCE.pm index 575959b7..4ea49b1a 100644 --- a/lib/App/Netdisco/Util/Backend.pm +++ b/lib/App/Netdisco/Util/MCE.pm @@ -1,4 +1,4 @@ -package App::Netdisco::Util::Backend; +package App::Netdisco::Util::MCE; use strict; use warnings; diff --git a/lib/App/Netdisco/Core/Nbtstat.pm b/lib/App/Netdisco/Util/Nbtstat.pm similarity index 91% rename from lib/App/Netdisco/Core/Nbtstat.pm rename to lib/App/Netdisco/Util/Nbtstat.pm index 4c66b1cf..c12efc56 100644 --- a/lib/App/Netdisco/Core/Nbtstat.pm +++ b/lib/App/Netdisco/Util/Nbtstat.pm @@ -1,10 +1,9 @@ -package App::Netdisco::Core::Nbtstat; +package App::Netdisco::Util::Nbtstat; use Dancer qw/:syntax :script/; use Dancer::Plugin::DBIC 'schema'; use App::Netdisco::Util::Node 'check_mac'; -use NetAddr::IP::Lite ':lower'; use App::Netdisco::AnyEvent::Nbtstat; use Encode; @@ -15,7 +14,7 @@ our %EXPORT_TAGS = (all => \@EXPORT_OK); =head1 NAME -App::Netdisco::Core::Nbtstat +App::Netdisco::Util::Nbtstat =head1 DESCRIPTION @@ -130,12 +129,12 @@ sub _filter_nbname { $mac = $node_ip->mac; } - $hash_ref->{'ip'} = $ip; - $hash_ref->{'mac'} = $mac; - $hash_ref->{'nbname'} = Encode::decode('UTF-8', $nbname); - $hash_ref->{'domain'} = Encode::decode('UTF-8', $domain); - $hash_ref->{'server'} = $server; - $hash_ref->{'nbuser'} = Encode::decode('UTF-8', $nbuser); + $hash_ref->{'ip'} = $ip; + $hash_ref->{'mac'} = $mac; + $hash_ref->{'nbname'} = Encode::decode('UTF-8', $nbname); + $hash_ref->{'domain'} = Encode::decode('UTF-8', $domain); + $hash_ref->{'server'} = $server; + $hash_ref->{'nbuser'} = Encode::decode('UTF-8', $nbuser); return; } diff --git a/lib/App/Netdisco/Util/SNMP.pm b/lib/App/Netdisco/Util/SNMP.pm index 791c11d8..f75dcf73 100644 --- a/lib/App/Netdisco/Util/SNMP.pm +++ b/lib/App/Netdisco/Util/SNMP.pm @@ -1,18 +1,13 @@ package App::Netdisco::Util::SNMP; use Dancer qw/:syntax :script/; -use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::DNS 'hostname_from_ip'; use App::Netdisco::Util::Permission ':all'; -use SNMP::Info; -use Try::Tiny; -use Module::Load (); -use Path::Class 'dir'; - use base 'Exporter'; our @EXPORT = (); our @EXPORT_OK = qw/ - snmp_connect snmp_connect_rw snmp_comm_reindex + fixup_device_auth get_communities snmp_comm_reindex /; our %EXPORT_TAGS = (all => \@EXPORT_OK); @@ -22,337 +17,139 @@ App::Netdisco::Util::SNMP =head1 DESCRIPTION -A set of helper subroutines to support parts of the Netdisco application. +Helper functions for L instances. There are no default exports, however the C<:all> tag will export all subroutines. =head1 EXPORT_OK -=head2 snmp_connect( $ip ) +=head2 fixup_device_auth -Given an IP address, returns an L instance configured for and -connected to that device. The IP can be any on the device, and the management -interface will be connected to. - -If the device is known to Netdisco and there is a cached SNMP community -string, this will be tried first, and then other community string(s) from the -application configuration will be tried. - -Returns C if the connection fails. +Rebuilds the C config with missing defaults and other fixups for +config changes over time. Returns a list which can replace C. =cut -sub snmp_connect { _snmp_connect_generic('read', @_) } - -=head2 snmp_connect_rw( $ip ) - -Same as C but uses the read-write community string(s) from the -application configuration file. - -Returns C if the connection fails. - -=cut - -sub snmp_connect_rw { _snmp_connect_generic('write', @_) } - -sub _snmp_connect_generic { - my ($mode, $ip, $useclass) = @_; - $mode ||= 'read'; - - # get device details from db - my $device = get_device($ip); - - my %snmp_args = ( - AutoSpecify => 0, - DestHost => $device->ip, - # 0 is falsy. Using || with snmpretries equal to 0 will set retries to 2. - # check if the setting is 0. If not, use the default value of 2. - Retries => (setting('snmpretries') || setting('snmpretries') == 0 ? 0 : 2), - Timeout => (setting('snmptimeout') || 1000000), - NonIncreasing => (setting('nonincreasing') || 0), - BulkWalk => ((defined setting('bulkwalk_off') && setting('bulkwalk_off')) - ? 0 : 1), - BulkRepeaters => (setting('bulkwalk_repeaters') || 20), - MibDirs => [ _build_mibdirs() ], - IgnoreNetSNMPConf => 1, - Debug => ($ENV{INFO_TRACE} || 0), - DebugSNMP => ($ENV{SNMP_TRACE} || 0), - ); - - # an override for bulkwalk - $snmp_args{BulkWalk} = 0 if check_acl_no($device, 'bulkwalk_no'); - - # further protect against buggy Net-SNMP, and disable bulkwalk - if ($snmp_args{BulkWalk} - and ($SNMP::VERSION eq '5.0203' || $SNMP::VERSION eq '5.0301')) { - - warning sprintf - "[%s] turning off BulkWalk due to buggy Net-SNMP - please upgrade!", - $device->ip; - $snmp_args{BulkWalk} = 0; - } - - # get the community string(s) - my @communities = _build_communities($device, $mode); - - # which SNMP versions to try and in what order - my @versions = - ( check_acl_no($device->ip, 'snmpforce_v3') ? (3) - : check_acl_no($device->ip, 'snmpforce_v2') ? (2) - : check_acl_no($device->ip, 'snmpforce_v1') ? (1) - : (reverse (1 .. (setting('snmpver') || 3))) ); - - # use existing or new device class - my @classes = ($useclass || 'SNMP::Info'); - if ($device->snmp_class and not $useclass) { - unshift @classes, $device->snmp_class; - } - - my $info = undef; - COMMUNITY: foreach my $comm (@communities) { - next unless $comm; - - VERSION: foreach my $ver (@versions) { - next unless $ver; - - next if $ver eq 3 and exists $comm->{community}; - next if $ver ne 3 and !exists $comm->{community}; - - CLASS: foreach my $class (@classes) { - next unless $class; - - my %local_args = (%snmp_args, Version => $ver); - $info = _try_connect($device, $class, $comm, $mode, \%local_args, - ($useclass ? 0 : 1) ); - last COMMUNITY if $info; - } - } - } - - return $info; -} - -sub _try_connect { - my ($device, $class, $comm, $mode, $snmp_args, $reclass) = @_; - my %comm_args = _mk_info_commargs($comm); - my $debug_comm = ''; - if ($ENV{SHOW_COMMUNITY}) { - $debug_comm = ($comm->{community} || - (sprintf 'v3:%s:%s/%s', ($comm->{user}, - ($comm->{auth}->{proto} || 'noAuth'), - ($comm->{priv}->{proto} || 'noPriv'))) ); - } - my $info = undef; - - try { - debug - sprintf '[%s] try_connect with ver: %s, class: %s, comm: %s', - $snmp_args->{DestHost}, $snmp_args->{Version}, $class, $debug_comm; - Module::Load::load $class; - - $info = $class->new(%$snmp_args, %comm_args) or return; - $info = ($mode eq 'read' ? _try_read($info, $device, $comm) - : _try_write($info, $device, $comm)); - - # first time a device is discovered, re-instantiate into specific class - if ($reclass and $info and $info->device_type ne $class) { - $class = $info->device_type; - debug - sprintf '[%s] try_connect with ver: %s, new class: %s, comm: %s', - $snmp_args->{DestHost}, $snmp_args->{Version}, $class, $debug_comm; - - Module::Load::load $class; - $info = $class->new(%$snmp_args, %comm_args); - } - } - catch { - debug $_; - }; - - return $info; -} - -sub _try_read { - my ($info, $device, $comm) = @_; - - return undef unless ( - (not defined $info->error) - and defined $info->uptime - and ($info->layers or $info->description) - and $info->class - ); - - $device->in_storage - ? $device->update({snmp_ver => $info->snmp_ver}) - : $device->set_column(snmp_ver => $info->snmp_ver); - - if ($comm->{community}) { - $device->in_storage - ? $device->update({snmp_comm => $comm->{community}}) - : $device->set_column(snmp_comm => $comm->{community}); - } - - # regardless of device in storage, save the hint - $device->update_or_create_related('community', - {snmp_auth_tag_read => $comm->{tag}}) if $comm->{tag}; - - return $info; -} - -sub _try_write { - my ($info, $device, $comm) = @_; - - my $loc = $info->load_location; - $info->set_location($loc) or return undef; - return undef unless ($loc eq $info->load_location); - - $device->in_storage - ? $device->update({snmp_ver => $info->snmp_ver}) - : $device->set_column(snmp_ver => $info->snmp_ver); - - # one of these two cols must be set - $device->update_or_create_related('community', { - ($comm->{tag} ? (snmp_auth_tag_write => $comm->{tag}) : ()), - ($comm->{community} ? (snmp_comm_rw => $comm->{community}) : ()), - }); - - return $info; -} - -sub _mk_info_commargs { - my $comm = shift; - return () unless ref {} eq ref $comm and scalar keys %$comm; - - return (Community => $comm->{community}) - if exists $comm->{community}; - - my $seclevel = - (exists $comm->{auth} ? - (exists $comm->{priv} ? 'authPriv' : 'authNoPriv' ) - : 'noAuthNoPriv'); - - return ( - SecName => $comm->{user}, - SecLevel => $seclevel, - ( exists $comm->{auth} ? ( - AuthProto => uc ($comm->{auth}->{proto} || 'MD5'), - AuthPass => ($comm->{auth}->{pass} || ''), - ( exists $comm->{priv} ? ( - PrivProto => uc ($comm->{priv}->{proto} || 'DES'), - PrivPass => ($comm->{priv}->{pass} || ''), - ) : ()), - ) : ()), - ); -} - -sub _build_mibdirs { - my $home = (setting('mibhome') || dir(($ENV{NETDISCO_HOME} || $ENV{HOME}), 'netdisco-mibs')); - return map { dir($home, $_)->stringify } - @{ setting('mibdirs') || _get_mibdirs_content($home) }; -} - -sub _get_mibdirs_content { - my $home = shift; - # warning 'Netdisco SNMP work will be slow - loading ALL MIBs. Consider setting mibdirs.'; - my @list = map {s|$home/||; $_} grep {-d} glob("$home/*"); - return \@list; -} - -sub _build_communities { - my ($device, $mode) = @_; - $mode ||= 'read'; - my $seen_tags = {}; # for cleaning community table - - my $config = (setting('snmp_auth') || []); - my $tag_name = 'snmp_auth_tag_'. $mode; - my $stored_tag = eval { $device->community->$tag_name }; - my $snmp_comm_rw = eval { $device->community->snmp_comm_rw }; - my @communities = (); - - # try last-known-good read - push @communities, {read => 1, community => $device->snmp_comm} - if defined $device->snmp_comm and $mode eq 'read'; - - # try last-known-good write - push @communities, {write => 1, community => $snmp_comm_rw} - if $snmp_comm_rw and $mode eq 'write'; +sub fixup_device_auth { + my $config = (setting('device_auth') || setting('snmp_auth') || []); + my @new_stanzas = (); # new style snmp config foreach my $stanza (@$config) { - # user tagged - my $tag = ''; - if (1 == scalar keys %$stanza) { - $tag = (keys %$stanza)[0]; - $stanza = $stanza->{$tag}; + # user tagged + my $tag = ''; + if (1 == scalar keys %$stanza) { + $tag = (keys %$stanza)[0]; + $stanza = $stanza->{$tag}; - # corner case: untagged lone community - if ($tag eq 'community') { - $tag = $stanza; - $stanza = {community => $tag}; - } + # corner case: untagged lone community + if ($tag eq 'community') { + $tag = $stanza; + $stanza = {community => $tag}; } + } - # defaults - $stanza->{tag} ||= $tag; - ++$seen_tags->{ $stanza->{tag} }; - $stanza->{read} = 1 if !exists $stanza->{read}; - $stanza->{no} ||= []; - $stanza->{only} ||= ['any']; - $stanza->{no} = [$stanza->{no}] if ref '' eq ref $stanza->{no}; - $stanza->{only} = [$stanza->{only}] if ref '' eq ref $stanza->{only}; + # defaults + $stanza->{tag} ||= $tag; + $stanza->{read} = 1 if !exists $stanza->{read}; + $stanza->{no} ||= []; + $stanza->{only} ||= ['any']; - die "error: config: snmpv2 community in snmp_auth must be single item, not list\n" - if ref $stanza->{community}; + die "error: config: snmpv2 community in device_auth must be single item, not list\n" + if ref $stanza->{community}; - die "error: config: snmpv3 stanza in snmp_auth must have a tag\n" - if not $stanza->{tag} - and !exists $stanza->{community}; + die "error: config: stanza in device_auth must have a tag\n" + if not $stanza->{tag} and exists $stanza->{user}; - if ($stanza->{$mode} and check_acl_only($device, $stanza->{only}) - and not check_acl_no($device, $stanza->{no})) { - if ($device->in_storage and - $stored_tag and $stored_tag eq $stanza->{tag}) { - # last known-good by tag - unshift @communities, $stanza - } - else { - push @communities, $stanza - } - } + push @new_stanzas, $stanza } - # clean the community table of obsolete tags - if ($stored_tag and !exists $seen_tags->{ $stored_tag }) { - eval { $device->community->update({$tag_name => undef}) }; + # legacy config + # note: read strings tried before write + # note: read-write is no longer used for read operations + + push @new_stanzas, map {{ + read => 1, write => 0, + no => [], only => ['any'], + community => $_, + }} @{setting('community') || []}; + + push @new_stanzas, map {{ + write => 1, read => 0, + no => [], only => ['any'], + community => $_, + }} @{setting('community_rw') || []}; + + foreach my $stanza (@new_stanzas) { + $stanza->{driver} ||= 'snmp' + if exists $stanza->{community} + or exists $stanza->{user}; } - # legacy config (note: read strings tried before write) - if ($mode eq 'read') { - push @communities, map {{ - read => 1, - community => $_, - }} @{setting('community') || []}; - } - else { - push @communities, map {{ - write => 1, - community => $_, - }} @{setting('community_rw') || []}; - } + return @new_stanzas; +} - # but first of all, use external command if configured - unshift @communities, _get_external_community($device, $mode) +=head2 get_communities( $device, $mode ) + +Takes the current C setting and pushes onto the front of the list +the last known good SNMP settings used for this mode (C or C). + +=cut + +sub get_communities { + my ($device, $mode) = @_; + $mode ||= 'read'; + + my $seen_tags = {}; # for cleaning community table + my $config = (setting('device_auth') || []); + my @communities = (); + + # first of all, use external command if configured + push @communities, _get_external_community($device, $mode) if setting('get_community') and length setting('get_community'); - return @communities; + # last known-good by tag + my $tag_name = 'snmp_auth_tag_'. $mode; + my $stored_tag = eval { $device->community->$tag_name }; + + if ($device->in_storage and $stored_tag) { + foreach my $stanza (@$config) { + if ($stanza->{tag} and $stored_tag eq $stanza->{tag}) { + push @communities, {%$stanza, only => [$device->ip]}; + last; + } + } + } + + # try last-known-good v2 read + push @communities, { + read => 1, write => 0, driver => 'snmp', + only => [$device->ip], + community => $device->snmp_comm, + } if defined $device->snmp_comm and $mode eq 'read'; + + # try last-known-good v2 write + my $snmp_comm_rw = eval { $device->community->snmp_comm_rw }; + push @communities, { + write => 1, read => 0, driver => 'snmp', + only => [$device->ip], + community => $snmp_comm_rw, + } if $snmp_comm_rw and $mode eq 'write'; + + # clean the community table of obsolete tags + eval { $device->community->update({$tag_name => undef}) } + if $device->in_storage + and (not $stored_tag or !exists $seen_tags->{ $stored_tag }); + + return ( @communities, @$config ); } sub _get_external_community { my ($device, $mode) = @_; my $cmd = setting('get_community'); my $ip = $device->ip; - my $host = $device->dns || $ip; + my $host = ($device->dns || hostname_from_ip($ip) || $ip); if (defined $cmd and length $cmd) { # replace variables @@ -368,6 +165,7 @@ sub _get_external_community { if (length $1 and $mode eq 'read') { return map {{ read => 1, + only => [$device->ip], community => $_, }} split(m/\s*,\s*/,$1); } @@ -376,6 +174,7 @@ sub _get_external_community { if (length $1 and $mode eq 'write') { return map {{ write => 1, + only => [$device->ip], community => $_, }} split(m/\s*,\s*/,$1); } @@ -391,6 +190,9 @@ sub _get_external_community { Takes an established L instance and makes a fresh connection using community indexing, with the given C<$vlan> ID. Works for all SNMP versions. +Passing VLAN "C<0>" (zero) will reset the indexing to the basic v2 community +or v3 empty context. + =cut sub snmp_comm_reindex { @@ -399,7 +201,8 @@ sub snmp_comm_reindex { if ($ver == 3) { my $prefix = ''; - my @comms = _build_communities($device, 'read'); + my @comms = get_communities($device, 'read'); + # find a context prefix configured by the user foreach my $c (@comms) { next unless $c->{tag} and $c->{tag} eq (eval { $device->community->snmp_auth_tag_read } || ''); @@ -410,15 +213,17 @@ sub snmp_comm_reindex { debug sprintf '[%s] reindexing to "%s%s" (ver: %s, class: %s)', $device->ip, $prefix, $vlan, $ver, $snmp->class; - $snmp->update(Context => ($prefix . $vlan)); + $vlan ? $snmp->update(Context => ($prefix . $vlan)) + : $snmp->update(Context => ''); } else { my $comm = $snmp->snmp_comm; debug sprintf '[%s] reindexing to vlan %s (ver: %s, class: %s)', $device->ip, $vlan, $ver, $snmp->class; - $snmp->update(Community => $comm . '@' . $vlan); + $vlan ? $snmp->update(Community => $comm . '@' . $vlan) + : $snmp->update(Community => $comm); } } -1; +true; diff --git a/lib/App/Netdisco/Web.pm b/lib/App/Netdisco/Web.pm index 05a177eb..06a68138 100644 --- a/lib/App/Netdisco/Web.pm +++ b/lib/App/Netdisco/Web.pm @@ -34,7 +34,7 @@ sub _load_web_plugins { if $plugin !~ m/^\+/; $plugin =~ s/^\+//; - debug "loading Netdisco plugin $plugin"; + $ENV{PLUGIN_LOAD_DEBUG} && debug "loading web plugin $plugin"; Module::Load::load $plugin; } } diff --git a/lib/App/Netdisco/Web/Plugin.pm b/lib/App/Netdisco/Web/Plugin.pm index 3b08229d..1a28e7dd 100644 --- a/lib/App/Netdisco/Web/Plugin.pm +++ b/lib/App/Netdisco/Web/Plugin.pm @@ -233,7 +233,8 @@ Admin Menu function (job control, manual topology, pseudo devices) =back This document explains how to configure which plugins are loaded. See -L if you want to develop new plugins. +L if you want to develop new +plugins. =head1 Application Configuration diff --git a/lib/App/Netdisco/Worker/Loader.pm b/lib/App/Netdisco/Worker/Loader.pm new file mode 100644 index 00000000..587c2903 --- /dev/null +++ b/lib/App/Netdisco/Worker/Loader.pm @@ -0,0 +1,55 @@ +package App::Netdisco::Worker::Loader; + +use strict; +use warnings; + +use Module::Load (); +use Dancer qw/:moose :syntax/; + +use Moo::Role; +use namespace::clean; + +has [qw/workers_check + workers_early + workers_main + workers_user/] => ( is => 'rw' ); + +sub load_workers { + my $self = shift; + my $action = $self->job->action or die "missing action\n"; + + my @core_plugins = @{ setting('worker_plugins') || [] }; + my @user_plugins = @{ setting('extra_worker_plugins') || [] }; + + # load worker plugins for our action + foreach my $plugin (@user_plugins, @core_plugins) { + $plugin =~ s/^X::/+App::NetdiscoX::Worker::Plugin::/; + $plugin = 'App::Netdisco::Worker::Plugin::'. $plugin + if $plugin !~ m/^\+/; + $plugin =~ s/^\+//; + + next unless $plugin =~ m/::Plugin::${action}(?:::|$)/i; + $ENV{PLUGIN_LOAD_DEBUG} && debug "loading worker plugin $plugin"; + Module::Load::load $plugin; + } + + # now vars->{workers} is populated, we set the dispatch order + my $workers = vars->{'workers'}->{$action} || {}; + #use DDP; p vars->{'workers'}; + + foreach my $phase (qw/check early main user/) { + my $pname = "workers_${phase}"; + my @wset = (); + + foreach my $namespace (sort keys %{ $workers->{$phase} }) { + foreach my $priority (sort {$b <=> $a} + keys %{ $workers->{$phase}->{$namespace} }) { + push @wset, @{ $workers->{$phase}->{$namespace}->{$priority} }; + } + } + + $self->$pname( \@wset ); + } +} + +true; diff --git a/lib/App/Netdisco/Worker/Plugin.pm b/lib/App/Netdisco/Worker/Plugin.pm new file mode 100644 index 00000000..16d19b0a --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin.pm @@ -0,0 +1,164 @@ +package App::Netdisco::Worker::Plugin; + +use Dancer ':syntax'; +use Dancer::Plugin; + +use App::Netdisco::Util::Permission qw/check_acl_no check_acl_only/; +use aliased 'App::Netdisco::Worker::Status'; +use Scope::Guard 'guard'; + +register 'register_worker' => sub { + my ($self, $first, $second) = plugin_args(@_); + + my $workerconf = (ref $first eq 'HASH' ? $first : {}); + my $code = (ref $first eq 'CODE' ? $first : $second); + return error "bad param to register_worker" + unless ((ref sub {} eq ref $code) and (ref {} eq ref $workerconf)); + + my $package = (caller)[0]; + if ($package =~ m/Plugin::(\w+)(?:::(\w+))?/) { + $workerconf->{action} = lc($1); + $workerconf->{namespace} = lc($2) if $2; + } + return error "failed to parse action in '$package'" + unless $workerconf->{action}; + + $workerconf->{phase} ||= 'user'; + $workerconf->{namespace} ||= '_base_'; + $workerconf->{priority} ||= (exists $workerconf->{driver} + ? (setting('driver_priority')->{$workerconf->{driver}} || 0) : 0); + + my $worker = sub { + my $job = shift or die 'missing job param'; + # use DDP; p $workerconf; + + # update job's record of namespace and priority + # check to see if this namespace has already passed at higher priority + return if $job->namespace_passed($workerconf); + + my @newuserconf = (); + my @userconf = @{ setting('device_auth') || [] }; + + # worker might be vendor/platform specific + if (ref $job->device) { + my $no = (exists $workerconf->{no} ? $workerconf->{no} : undef); + my $only = (exists $workerconf->{only} ? $workerconf->{only} : undef); + + return $job->add_status( Status->noop('worker not applicable to this device') ) + if ($no and check_acl_no($job->device, $no)) + or ($only and not check_acl_only($job->device, $only)); + + # reduce device_auth by driver and action filters + foreach my $stanza (@userconf) { + next if exists $stanza->{driver} and exists $workerconf->{driver} + and (($stanza->{driver} || '') ne ($workerconf->{driver} || '')); + + # filter here rather than in Runner as runner does not know namespace + next if exists $stanza->{action} + and not _find_matchaction($workerconf, lc($stanza->{action})); + + push @newuserconf, $stanza; + } + + # per-device action but no device creds available + return $job->add_status( Status->noop('worker driver or action not applicable') ) + if 0 == scalar @newuserconf; + } + + # back up and restore device_auth + my $guard = guard { set(device_auth => \@userconf) }; + set(device_auth => \@newuserconf); + # use DDP; p @newuserconf; + + # run worker + $code->($job, $workerconf); + }; + + # store the built worker as Worker.pm will build the dispatch order later on + push @{ vars->{'workers'}->{$workerconf->{action}} + ->{$workerconf->{phase}} + ->{$workerconf->{namespace}} + ->{$workerconf->{priority}} }, $worker; +}; + +sub _find_matchaction { + my ($conf, $action) = @_; + return true if !defined $action; + $action = [$action] if ref [] ne ref $action; + + foreach my $f (@$action) { + return true if + $f eq $conf->{action} or $f eq "$conf->{action}::$conf->{namespace}"; + } + return false; +} + +register_plugin; +true; + +=head1 NAME + +App::Netdisco::Worker::Plugin - Netdisco Workers + +=head1 Introduction + +L's plugin system allows users to write I to gather +information from network devices using different I and store +results in the database. + +For example, transports might be SNMP, SSH, or HTTPS. Workers might be +combining those transports with application protocols such as SNMP, NETCONF +(OpenConfig with XML), RESTCONF (OpenConfig with JSON), eAPI, or even CLI +scraping. The combination of transport and protocol is known as a I. + +Workers can be restricted to certain vendor platforms using familiar ACL +syntax. They are also attached to specific actions in Netdisco's backend +operation (discover, macsuck, etc). + +=head1 Application Configuration + +The C and C settings list in YAML format +the set of Perl module names which are the plugins to be loaded. + +Any change should go into your local C configuration file. If +you want to view the default settings, see the C file in the +C distribution. + +=head1 How to Configure + +The C setting is empty, and used when you want to add +new plugins and not change the set enabled by default. If you do want to add +to or remove from the default set, then create a version of C +instead. + +Netdisco prepends "C" to any entry in the +list. For example, "C" will load the +C package. + +You can prepend module names with "C" as shorthand for the "Netdisco +extension" namespace. For example, "C" will +load the L +module. + +If an entry in the list starts with a "C<+>" (plus) sign then Netdisco attemps +to load the module as-is, without prepending anything to the name. This allows +you to have worker plugins in any namespace. + +Plugin modules can either ship with the App::Netdisco distribution itself, or +be installed separately. Perl uses the standard C<@INC> path searching +mechanism to load the plugin modules. See the C and +C settings in order to modify C<@INC> for loading local +plugins. + +As an example, if you set C to be true, set +C to be C<'X::MyPluginName'> (the plugin package is +"App::NetdiscoX::Worker::Plugin::MyPluginName") then your plugin lives at: + + ~netdisco/nd-site-local/lib/App/NetdiscoX/Worker/Plugin/MyPluginName.pm + +The order of the entries is significant, workers being executed in the order +which they appear in C followed by C. + +See L for further details. +=cut + diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip.pm new file mode 100644 index 00000000..1d79f128 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::Arpnip; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Device 'is_arpnipable_now'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + my $device = $job->device; + + return Status->error('arpnip failed: unable to interpret device param') + unless defined $device; + + return Status->error("arpnip skipped: $device not yet discovered") + unless $device->in_storage; + + return Status->defer("arpnip skipped: $device is pseudo-device") + if $device->is_pseudo; + + return Status->defer("arpnip skipped: $device has no layer 3 capability") + unless $device->has_layer(3); + + return Status->defer("arpnip deferred: $device is not arpnipable") + unless is_arpnipable_now($device); + + return Status->done('arpnip is able to run'); +}); + +true; diff --git a/lib/App/Netdisco/Core/Arpnip.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm similarity index 52% rename from lib/App/Netdisco/Core/Arpnip.pm rename to lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm index 335fff87..4c5e82a4 100644 --- a/lib/App/Netdisco/Core/Arpnip.pm +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Nodes.pm @@ -1,58 +1,27 @@ -package App::Netdisco::Core::Arpnip; +package App::Netdisco::Worker::Plugin::Arpnip::Nodes; -use Dancer qw/:syntax :script/; -use Dancer::Plugin::DBIC 'schema'; +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; +use App::Netdisco::Transport::SNMP (); use App::Netdisco::Util::Node 'check_mac'; -use App::Netdisco::Util::Permission 'check_acl_no'; use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; -use NetAddr::IP::Lite ':lower'; +use Dancer::Plugin::DBIC 'schema'; use Time::HiRes 'gettimeofday'; use NetAddr::MAC (); -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ do_arpnip store_arp /; -our %EXPORT_TAGS = (all => \@EXPORT_OK); +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; -=head1 NAME - -App::Netdisco::Core::Arpnip - -=head1 DESCRIPTION - -Helper subroutines to support parts of the Netdisco application. - -There are no default exports, however the C<:all> tag will export all -subroutines. - -=head1 EXPORT_OK - -=head2 do_arpnip( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, connect to a -device and discover its ARP cache for IPv4 and Neighbor cache for IPv6. - -Will also discover subnets in use on the device and update the Subnets table. - -=cut - -sub do_arpnip { - my ($device, $snmp) = @_; - - unless ($device->in_storage) { - debug sprintf ' [%s] arpnip - skipping device not yet discovered', $device->ip; - return; - } + my $device = $job->device; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("arpnip failed: could not SNMP connect to $device"); # get v4 arp table - my $v4 = _get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); + my $v4 = get_arps($device, $snmp->at_paddr, $snmp->at_netaddr); # get v6 neighbor cache - my $v6 = _get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); - - # get directly connected networks - my @subnets = _gather_subnets($device, $snmp); - # TODO: IPv6 subnets + my $v6 = get_arps($device, $snmp->ipv6_n2p_mac, $snmp->ipv6_n2p_addr); # would be possible just to use now() on updated records, but by using this # same value for them all, we _can_ if we want add a job at the end to @@ -68,15 +37,12 @@ sub do_arpnip { debug sprintf ' [%s] arpnip - processed %s IPv6 Neighbor Cache entries', $device->ip, scalar @$v6; - _store_subnet($_, $now) for @subnets; - debug sprintf ' [%s] arpnip - processed %s Subnet entries', - $device->ip, scalar @subnets; - $device->update({last_arpnip => \$now}); -} + return Status->done("Ended arpnip for $device"); +}); # get an arp table (v4 or v6) -sub _get_arps { +sub get_arps { my ($device, $paddr, $netaddr) = @_; my @arps = (); @@ -92,7 +58,7 @@ sub _get_arps { } debug sprintf ' resolving %d ARP entries with max %d outstanding requests', - scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; + scalar @arps, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; my $resolved_ips = hostnames_resolve_async(\@arps); return $resolved_ips; @@ -148,44 +114,4 @@ sub store_arp { }); } -# gathers device subnets -sub _gather_subnets { - my ($device, $snmp) = @_; - my @subnets = (); - - my $ip_netmask = $snmp->ip_netmask; - foreach my $entry (keys %$ip_netmask) { - my $ip = NetAddr::IP::Lite->new($entry); - my $addr = $ip->addr; - - next if $addr eq '0.0.0.0'; - next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__'); - next if setting('ignore_private_nets') and $ip->is_rfc1918; - - my $netmask = $ip_netmask->{$addr}; - next if $netmask eq '255.255.255.255' or $netmask eq '0.0.0.0'; - - my $cidr = NetAddr::IP::Lite->new($addr, $netmask)->network->cidr; - - debug sprintf ' [%s] arpnip - found subnet %s', $device->ip, $cidr; - push @subnets, $cidr; - } - - return @subnets; -} - -# update subnets with new networks -sub _store_subnet { - my ($subnet, $now) = @_; - - schema('netdisco')->txn_do(sub { - schema('netdisco')->resultset('Subnet')->update_or_create( - { - net => $subnet, - last_discover => \$now, - }, - { for => 'update' }); - }); -} - -1; +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm b/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm new file mode 100644 index 00000000..0dd6f256 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Arpnip/Subnets.pm @@ -0,0 +1,74 @@ +package App::Netdisco::Worker::Plugin::Arpnip::Subnets; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use App::Netdisco::Util::Permission 'check_acl_no'; +use Dancer::Plugin::DBIC 'schema'; +use NetAddr::IP::Lite ':lower'; +use Time::HiRes 'gettimeofday'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("arpnip failed: could not SNMP connect to $device"); + + # get directly connected networks + my @subnets = gather_subnets($device); + # TODO: IPv6 subnets + + my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; + store_subnet($_, $now) for @subnets; + + return Status->noop(sprintf ' [%s] arpnip - processed %s Subnet entries', + $device->ip, scalar @subnets); +}); + +# gathers device subnets +sub gather_subnets { + my $device = shift; + my @subnets = (); + + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return (); # already checked! + + my $ip_netmask = $snmp->ip_netmask; + foreach my $entry (keys %$ip_netmask) { + my $ip = NetAddr::IP::Lite->new($entry); + my $addr = $ip->addr; + + next if $addr eq '0.0.0.0'; + next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__'); + next if setting('ignore_private_nets') and $ip->is_rfc1918; + + my $netmask = $ip_netmask->{$addr}; + next if $netmask eq '255.255.255.255' or $netmask eq '0.0.0.0'; + + my $cidr = NetAddr::IP::Lite->new($addr, $netmask)->network->cidr; + + debug sprintf ' [%s] arpnip - found subnet %s', $device->ip, $cidr; + push @subnets, $cidr; + } + + return @subnets; +} + +# update subnets with new networks +sub store_subnet { + my ($subnet, $now) = @_; + + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('Subnet')->update_or_create( + { + net => $subnet, + last_discover => \$now, + }, + { for => 'update' }); + }); +} + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm new file mode 100644 index 00000000..599aba4c --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Arpwalk.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::Arpwalk; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + + my %queued = map {$_ => 1} jq_queued('arpnip'); + my @devices = schema('netdisco')->resultset('Device')->search({ + -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], + })->has_layer('3')->get_column('ip')->all; + my @filtered_devices = grep {!exists $queued{$_}} @devices; + + jq_insert([ + map {{ + device => $_, + action => 'arpnip', + username => $job->username, + userip => $job->userip, + }} (@filtered_devices) + ]); + + return Status->done('Queued arpnip job for all devices'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Contact.pm b/lib/App/Netdisco/Worker/Plugin/Contact.pm new file mode 100644 index 00000000..6e059cbf --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Contact.pm @@ -0,0 +1,43 @@ +package App::Netdisco::Worker::Plugin::Contact; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + return Status->done('Contact is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $data) = map {$job->$_} qw/device extra/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update contact"); + + my $rv = $snmp->set_contact($data); + + if (!defined $rv) { + return Status->error( + "failed to set contact on $device: ". ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $new_data = ($snmp->contact || ''); + if ($new_data ne $data) { + return Status->error("verify of contact failed on $device: $new_data"); + } + + # update netdisco DB + $device->update({contact => $data}); + + return Status->done("Updated contact on $device to [$data]"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Delete.pm b/lib/App/Netdisco/Worker/Plugin/Delete.pm new file mode 100644 index 00000000..8ecb61ca --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Delete.pm @@ -0,0 +1,24 @@ +package App::Netdisco::Worker::Plugin::Delete; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Device 'delete_device'; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + return Status->done('Delete is able to run'); +}); + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; + + $port = ($port ? 1 : 0); + delete_device($device, $port, $extra); + return Status->done("Deleted device: $device"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover.pm b/lib/App/Netdisco/Worker/Plugin/Discover.pm new file mode 100644 index 00000000..96c4dc05 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover.pm @@ -0,0 +1,28 @@ +package App::Netdisco::Worker::Plugin::Discover; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Device 'is_discoverable_now'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + my $device = $job->device; + + return Status->error('discover failed: unable to interpret device param') + unless defined $device; + + return Status->error("discover failed: no device param (need -d ?)") + if $device->ip eq '0.0.0.0'; + + return Status->defer("discover skipped: $device is pseudo-device") + if $device->is_pseudo; + + return Status->defer("discover deferred: $device is not discoverable") + unless is_discoverable_now($device); + + return Status->done('Discover is able to run.'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm b/lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm new file mode 100644 index 00000000..105f4185 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/CanonicalIP.pm @@ -0,0 +1,80 @@ +package App::Netdisco::Worker::Plugin::Discover::CanonicalIP; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use App::Netdisco::Util::Permission 'check_acl_only'; +use App::Netdisco::Util::DNS 'ipv4_from_hostname'; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $old_ip = $device->ip; + my $new_ip = $old_ip; + my $revofname = ipv4_from_hostname($snmp->name); + + if (setting('reverse_sysname') and $revofname) { + if (App::Netdisco::Transport::SNMP->test_connection( $new_ip )) { + $new_ip = $revofname; + } + else { + debug sprintf ' [%s] device - cannot renumber to %s - SNMP connect failed', + $old_ip, $revofname; + } + } + + if (scalar @{ setting('device_identity') }) { + my @idmaps = @{ setting('device_identity') }; + my $devips = $device->device_ips->order_by('alias'); + + ALIAS: while (my $alias = $devips->next) { + next if $alias->alias eq $old_ip; + + foreach my $map (@idmaps) { + next unless ref {} eq ref $map; + + foreach my $key (sort keys %$map) { + # lhs matches device, rhs matches device_ip + if (check_acl_only($device, $key) + and check_acl_only($alias, $map->{$key})) { + + if (App::Netdisco::Transport::SNMP->test_connection( $alias->alias )) { + $new_ip = $alias->alias; + last ALIAS; + } + else { + debug sprintf ' [%s] device - cannot renumber to %s - SNMP connect failed', + $old_ip, $alias->alias; + } + } + } + } + } # ALIAS + } + + return if $new_ip eq $old_ip; + + schema('netdisco')->txn_do(sub { + # delete target device with the same vendor and serial number + schema('netdisco')->resultset('Device')->search({ + ip => $new_ip, vendor => $device->vendor, serial => $device->serial, + })->delete; + + # if target device exists then this will die + $device->renumber($new_ip) + or die "cannot renumber to: $new_ip"; # rollback + + return Status->noop(sprintf ' [%s] device - changed IP to %s (%s)', + $old_ip, $device->ip, ($device->dns || '')); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm new file mode 100644 index 00000000..f3ee0d7a --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Entities.pm @@ -0,0 +1,95 @@ +package App::Netdisco::Worker::Plugin::Discover::Entities; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use Dancer::Plugin::DBIC 'schema'; +use Encode; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $e_index = $snmp->e_index; + + if (!defined $e_index) { + schema('netdisco')->txn_do(sub { + my $gone = $device->modules->delete; + debug sprintf ' [%s] modules - removed %d chassis modules', + $device->ip, $gone; + + $device->modules->update_or_create({ + ip => $device->ip, + index => 1, + parent => 0, + name => 'chassis', + class => 'chassis', + pos => -1, + # too verbose and link doesn't work anyway + # description => $device->description, + sw_ver => $device->os_ver, + serial => $device->serial, + model => $device->model, + fru => \'false', + last_discover => \'now()', + }); + }); + + return Status->noop( + sprintf ' [%s] modules - 0 chassis components (added one pseudo for chassis)', + $device->ip); + } + + my $e_descr = $snmp->e_descr; + my $e_type = $snmp->e_type; + my $e_parent = $snmp->e_parent; + my $e_name = $snmp->e_name; + my $e_class = $snmp->e_class; + my $e_pos = $snmp->e_pos; + my $e_hwver = $snmp->e_hwver; + my $e_fwver = $snmp->e_fwver; + my $e_swver = $snmp->e_swver; + my $e_model = $snmp->e_model; + my $e_serial = $snmp->e_serial; + my $e_fru = $snmp->e_fru; + + # build device modules list for DBIC + my (@modules, %seen_idx); + foreach my $entry (keys %$e_index) { + next if $seen_idx{ $e_index->{$entry} }++; + push @modules, { + index => $e_index->{$entry}, + type => $e_type->{$entry}, + parent => $e_parent->{$entry}, + name => Encode::decode('UTF-8', $e_name->{$entry}), + class => $e_class->{$entry}, + pos => $e_pos->{$entry}, + hw_ver => Encode::decode('UTF-8', $e_hwver->{$entry}), + fw_ver => Encode::decode('UTF-8', $e_fwver->{$entry}), + sw_ver => Encode::decode('UTF-8', $e_swver->{$entry}), + model => Encode::decode('UTF-8', $e_model->{$entry}), + serial => Encode::decode('UTF-8', $e_serial->{$entry}), + fru => $e_fru->{$entry}, + description => Encode::decode('UTF-8', $e_descr->{$entry}), + last_discover => \'now()', + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->modules->delete; + debug sprintf ' [%s] modules - removed %d chassis modules', + $device->ip, $gone; + $device->modules->populate(\@modules); + + return Status->noop(sprintf ' [%s] modules - added %d new chassis modules', + $device->ip, scalar @modules); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm new file mode 100644 index 00000000..aecec0e9 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Neighbors.pm @@ -0,0 +1,338 @@ +package App::Netdisco::Worker::Plugin::Discover::Neighbors; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use App::Netdisco::Util::Device + qw/get_device match_devicetype is_discoverable/; +use App::Netdisco::Util::Permission 'check_acl_no'; +use App::Netdisco::JobQueue 'jq_insert'; +use Dancer::Plugin::DBIC 'schema'; +use List::MoreUtils (); +use NetAddr::MAC; +use Encode; + +=head2 discover_new_neighbors( ) + +Given a Device database object, and a working SNMP connection, discover and +store the device's port neighbors information. + +Entries in the Topology database table will override any discovered device +port relationships. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +Any discovered neighbor unknown to Netdisco will have a C job +immediately queued (subject to the filtering by the C settings). + +=cut + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my @to_discover = store_neighbors($device); + + # only enqueue if device is not already discovered, + # discover_* config permits the discovery + foreach my $neighbor (@to_discover) { + my ($ip, $remote_type) = @$neighbor; + + my $device = get_device($ip); + next if $device->in_storage; + + if (not is_discoverable($device, $remote_type)) { + debug sprintf + ' queue - %s, type [%s] excluded by discover_* config', + $ip, ($remote_type || ''); + next; + } + + jq_insert({ + device => $ip, + action => 'discover', + subaction => 'with-nodes', + }); + } +}); + +=head2 store_neighbors( $device ) + +returns: C<@to_discover> + +Given a Device database object, and a working SNMP connection, discover and +store the device's port neighbors information. + +Entries in the Topology database table will override any discovered device +port relationships. + +The Device database object can be a fresh L object which is +not yet stored to the database. + +A list of discovererd neighbors will be returned as [C<$ip>, C<$type>] tuples. + +=cut + +sub store_neighbors { + my $device = shift; + my @to_discover = (); + + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return (); # already checked! + + # first allow any manually configured topology to be set + set_manual_topology($device); + + if (!defined $snmp->has_topo) { + debug sprintf ' [%s] neigh - neighbor protocols are not enabled', $device->ip; + return @to_discover; + } + + my $interfaces = $snmp->interfaces; + my $c_if = $snmp->c_if; + my $c_port = $snmp->c_port; + my $c_id = $snmp->c_id; + my $c_platform = $snmp->c_platform; + my $c_cap = $snmp->c_cap; + + # v4 and v6 neighbor tables + my $c_ip = ($snmp->c_ip || {}); + my %c_ipv6 = %{ ($snmp->can('hasLLDP') and $snmp->hasLLDP) + ? ($snmp->lldp_ipv6 || {}) : {} }; + + # remove keys with undef values, as c_ip does + delete @c_ipv6{ grep { not defined $c_ipv6{$_} } keys %c_ipv6 }; + # now combine them, v6 wins + $c_ip = { %$c_ip, %c_ipv6 }; + + foreach my $entry (sort (List::MoreUtils::uniq( (keys %$c_ip), (keys %$c_cap) ))) { + if (!defined $c_if->{$entry} or !defined $interfaces->{ $c_if->{$entry} }) { + debug sprintf ' [%s] neigh - port for IID:%s not resolved, skipping', + $device->ip, $entry; + next; + } + + my $port = $interfaces->{ $c_if->{$entry} }; + my $portrow = schema('netdisco')->resultset('DevicePort') + ->single({ip => $device->ip, port => $port}); + + if (!defined $portrow) { + info sprintf ' [%s] neigh - local port %s not in database!', + $device->ip, $port; + next; + } + + if (ref $c_ip->{$entry}) { + error sprintf ' [%s] neigh - Error! port %s has multiple neighbors - skipping', + $device->ip, $port; + next; + } + + my $remote_ip = $c_ip->{$entry}; + my $remote_port = undef; + my $remote_type = Encode::decode('UTF-8', $c_platform->{$entry} || ''); + my $remote_id = Encode::decode('UTF-8', $c_id->{$entry}); + my $remote_cap = $c_cap->{$entry} || []; + + # IP Phone and WAP detection type fixup + if (scalar @$remote_cap or $remote_type) { + my $phone_flag = grep {match_devicetype($_, 'phone_capabilities')} + @$remote_cap; + my $ap_flag = grep {match_devicetype($_, 'wap_capabilities')} + @$remote_cap; + + if ($phone_flag or match_devicetype($remote_type, 'phone_platforms')) { + $remote_type = 'IP Phone: '. $remote_type + if $remote_type !~ /ip.phone/i; + } + elsif ($ap_flag or match_devicetype($remote_type, 'wap_platforms')) { + $remote_type = 'AP: '. $remote_type; + } + + $portrow->update({remote_type => $remote_type}); + } + + if ($portrow->manual_topo) { + info sprintf ' [%s] neigh - %s has manually defined topology', + $device->ip, $port; + next; + } + + next unless $remote_ip; + + # a bunch of heuristics to search known devices if we don't have a + # useable remote IP... + + if ($remote_ip eq '0.0.0.0' or + check_acl_no($remote_ip, 'group:__LOCAL_ADDRESSES__')) { + + if ($remote_id) { + my $devices = schema('netdisco')->resultset('Device'); + my $neigh = $devices->single({name => $remote_id}); + info sprintf + ' [%s] neigh - bad address %s on port %s, searching for %s instead', + $device->ip, $remote_ip, $port, $remote_id; + + if (!defined $neigh) { + my $mac = NetAddr::MAC->new(mac => $remote_id); + if ($mac and not $mac->errstr) { + $neigh = $devices->single({mac => $mac->as_ieee}); + } + } + + # some HP switches send 127.0.0.1 as remote_ip if no ip address + # on default vlan for HP switches remote_ip looks like + # "myswitchname(012345-012345)" + if (!defined $neigh) { + (my $tmpid = $remote_id) =~ s/.*\(([0-9a-f]{6})-([0-9a-f]{6})\).*/$1$2/; + my $mac = NetAddr::MAC->new(mac => $tmpid); + if ($mac and not $mac->errstr) { + info sprintf + '[%s] neigh - found neighbor %s by MAC %s', + $device->ip, $remote_id, $mac->as_ieee; + $neigh = $devices->single({mac => $mac->as_ieee}); + } + } + + if (!defined $neigh) { + (my $shortid = $remote_id) =~ s/\..*//; + $neigh = $devices->single({name => { -ilike => "${shortid}%" }}); + } + + if ($neigh) { + $remote_ip = $neigh->ip; + info sprintf ' [%s] neigh - found %s with IP %s', + $device->ip, $remote_id, $remote_ip; + } + else { + info sprintf ' [%s] neigh - could not find %s, skipping', + $device->ip, $remote_id; + next; + } + } + else { + info sprintf ' [%s] neigh - skipping unuseable address %s on port %s', + $device->ip, $remote_ip, $port; + next; + } + } + + # what we came here to do.... discover the neighbor + debug sprintf + ' [%s] neigh - adding neighbor %s, type [%s], on %s to discovery queue', + $device->ip, $remote_ip, ($remote_type || ''), $port; + push @to_discover, [$remote_ip, $remote_type]; + + $remote_port = $c_port->{$entry}; + if (defined $remote_port) { + # clean weird characters + $remote_port =~ s/[^\d\/\.,()\w:-]+//gi; + } + else { + info sprintf ' [%s] neigh - no remote port found for port %s at %s', + $device->ip, $port, $remote_ip; + } + + $portrow->update({ + remote_ip => $remote_ip, + remote_port => $remote_port, + remote_type => $remote_type, + remote_id => $remote_id, + is_uplink => \"true", + manual_topo => \"false", + }); + + # update master of our aggregate to be a neighbor of + # the master on our peer device (a lot of iffs to get there...). + # & cannot use ->neighbor prefetch because this is the port insert! + if (defined $portrow->slave_of) { + + my $peer_device = get_device($remote_ip); + my $master = schema('netdisco')->resultset('DevicePort')->single({ + ip => $device->ip, + port => $portrow->slave_of + }); + + if ($peer_device and $peer_device->in_storage and $master + and not ($portrow->is_master or defined $master->slave_of)) { + + my $peer_port = schema('netdisco')->resultset('DevicePort')->single({ + ip => $peer_device->ip, + port => $portrow->remote_port, + }); + + $master->update({ + remote_ip => ($peer_device->ip || $remote_ip), + remote_port => ($peer_port ? $peer_port->slave_of : undef ), + is_uplink => \"true", + is_master => \"true", + manual_topo => \"false", + }); + } + } + } + + return @to_discover; +} + +# take data from the topology table and update remote_ip and remote_port +# in the devices table. only use root_ips and skip any bad topo entries. +sub set_manual_topology { + my $device = shift; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) or return; + + schema('netdisco')->txn_do(sub { + # clear manual topology flags + schema('netdisco')->resultset('DevicePort') + ->search({ip => $device->ip})->update({manual_topo => \'false'}); + + my $topo_links = schema('netdisco')->resultset('Topology') + ->search({-or => [dev1 => $device->ip, dev2 => $device->ip]}); + debug sprintf ' [%s] neigh - setting manual topology links', $device->ip; + + while (my $link = $topo_links->next) { + # could fail for broken topo, but we ignore to try the rest + try { + schema('netdisco')->txn_do(sub { + # only work on root_ips + my $left = get_device($link->dev1); + my $right = get_device($link->dev2); + + # skip bad entries + return unless ($left->in_storage and $right->in_storage); + + $left->ports + ->single({port => $link->port1}) + ->update({ + remote_ip => $right->ip, + remote_port => $link->port2, + remote_type => undef, + remote_id => undef, + is_uplink => \"true", + manual_topo => \"true", + }); + + $right->ports + ->single({port => $link->port2}) + ->update({ + remote_ip => $left->ip, + remote_port => $link->port1, + remote_type => undef, + remote_id => undef, + is_uplink => \"true", + manual_topo => \"true", + }); + }); + }; + } + }); +} + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm b/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm new file mode 100644 index 00000000..af122da1 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/PortPower.pm @@ -0,0 +1,81 @@ +package App::Netdisco::Worker::Plugin::Discover::PortPower; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $p_watts = $snmp->peth_power_watts; + my $p_status = $snmp->peth_power_status; + + if (!defined $p_watts) { + return Status->noop(sprintf ' [%s] power - 0 power modules', $device->ip); + } + + # build device module power info suitable for DBIC + my @devicepower; + foreach my $entry (keys %$p_watts) { + push @devicepower, { + module => $entry, + power => $p_watts->{$entry}, + status => $p_status->{$entry}, + }; + } + + my $interfaces = $snmp->interfaces; + my $p_ifindex = $snmp->peth_port_ifindex; + my $p_admin = $snmp->peth_port_admin; + my $p_pstatus = $snmp->peth_port_status; + my $p_class = $snmp->peth_port_class; + my $p_power = $snmp->peth_port_power; + + # build device port power info suitable for DBIC + my @portpower; + foreach my $entry (keys %$p_ifindex) { + my $port = $interfaces->{ $p_ifindex->{$entry} }; + next unless $port; + + my ($module) = split m/\./, $entry; + + push @portpower, { + port => $port, + module => $module, + admin => $p_admin->{$entry}, + status => $p_pstatus->{$entry}, + class => $p_class->{$entry}, + power => $p_power->{$entry}, + + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->power_modules->delete; + debug sprintf ' [%s] power - removed %d power modules', + $device->ip, $gone; + $device->power_modules->populate(\@devicepower); + debug sprintf ' [%s] power - added %d new power modules', + $device->ip, scalar @devicepower; + }); + + schema('netdisco')->txn_do(sub { + my $gone = $device->powered_ports->delete; + debug sprintf ' [%s] power - removed %d PoE capable ports', + $device->ip, $gone; + $device->powered_ports->populate(\@portpower); + + return Status->noop(sprintf ' [%s] power - added %d new PoE capable ports', + $device->ip, scalar @portpower); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm new file mode 100644 index 00000000..b8a50ab7 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Properties.pm @@ -0,0 +1,245 @@ +package App::Netdisco::Worker::Plugin::Discover::Properties; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use App::Netdisco::Util::Permission 'check_acl_no'; +use App::Netdisco::Util::FastResolver 'hostnames_resolve_async'; +use App::Netdisco::Util::DNS 'hostname_from_ip'; +use Dancer::Plugin::DBIC 'schema'; +use NetAddr::IP::Lite ':lower'; +use Encode; + +register_worker({ phase => 'early', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $ip_index = $snmp->ip_index; + my $interfaces = $snmp->interfaces; + my $ip_netmask = $snmp->ip_netmask; + + # build device aliases suitable for DBIC + my @aliases; + foreach my $entry (keys %$ip_index) { + my $ip = NetAddr::IP::Lite->new($entry) + or next; + my $addr = $ip->addr; + + next if $addr eq '0.0.0.0'; + next if check_acl_no($ip, 'group:__LOCAL_ADDRESSES__'); + next if setting('ignore_private_nets') and $ip->is_rfc1918; + + my $iid = $ip_index->{$addr}; + my $port = $interfaces->{$iid}; + my $subnet = $ip_netmask->{$addr} + ? NetAddr::IP::Lite->new($addr, $ip_netmask->{$addr})->network->cidr + : undef; + + debug sprintf ' [%s] device - aliased as %s', $device->ip, $addr; + push @aliases, { + alias => $addr, + port => $port, + subnet => $subnet, + dns => undef, + }; + } + + debug sprintf ' resolving %d aliases with max %d outstanding requests', + scalar @aliases, $ENV{'PERL_ANYEVENT_MAX_OUTSTANDING_DNS'}; + my $resolved_aliases = hostnames_resolve_async(\@aliases); + + # fake one aliases entry for devices not providing ip_index + push @$resolved_aliases, { alias => $device->ip, dns => $device->dns } + if 0 == scalar @aliases; + + # VTP Management Domain -- assume only one. + my $vtpdomains = $snmp->vtp_d_name; + my $vtpdomain; + if (defined $vtpdomains and scalar values %$vtpdomains) { + $device->set_column( vtp_domain => (values %$vtpdomains)[-1] ); + } + + my $hostname = hostname_from_ip($device->ip); + $device->set_column( dns => $hostname ) if $hostname; + + my @properties = qw/ + snmp_ver + description uptime name + layers ports mac + ps1_type ps2_type ps1_status ps2_status + fan slots + vendor os os_ver + /; + + foreach my $property (@properties) { + $device->set_column( $property => $snmp->$property ); + } + + $device->set_column( model => Encode::decode('UTF-8', $snmp->model) ); + $device->set_column( serial => Encode::decode('UTF-8', $snmp->serial) ); + $device->set_column( contact => Encode::decode('UTF-8', $snmp->contact) ); + $device->set_column( location => Encode::decode('UTF-8', $snmp->location) ); + + + $device->set_column( snmp_class => $snmp->class ); + $device->set_column( last_discover => \'now()' ); + + schema('netdisco')->txn_do(sub { + my $gone = $device->device_ips->delete; + debug sprintf ' [%s] device - removed %d aliases', + $device->ip, $gone; + $device->update_or_insert(undef, {for => 'update'}); + $device->device_ips->populate($resolved_aliases); + debug sprintf ' [%s] device - added %d new aliases', + $device->ip, scalar @aliases; + }); + + return Status->done("Ended discover for $device"); +}); + +register_worker({ phase => 'early', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $interfaces = $snmp->interfaces; + my $i_type = $snmp->i_type; + my $i_ignore = $snmp->i_ignore; + my $i_descr = $snmp->i_description; + my $i_mtu = $snmp->i_mtu; + my $i_speed = $snmp->i_speed; + my $i_mac = $snmp->i_mac; + my $i_up = $snmp->i_up; + my $i_up_admin = $snmp->i_up_admin; + my $i_name = $snmp->i_name; + my $i_duplex = $snmp->i_duplex; + my $i_duplex_admin = $snmp->i_duplex_admin; + my $i_stp_state = $snmp->i_stp_state; + my $i_vlan = $snmp->i_vlan; + my $i_lastchange = $snmp->i_lastchange; + my $agg_ports = $snmp->agg_ports; + + # clear the cached uptime and get a new one + my $dev_uptime = $snmp->load_uptime; + if (!defined $dev_uptime) { + error sprintf ' [%s] interfaces - Error! Failed to get uptime from device!', + $device->ip; + return Status->error("discover failed: no uptime from device $device!"); + } + + # used to track how many times the device uptime wrapped + my $dev_uptime_wrapped = 0; + + # use SNMP-FRAMEWORK-MIB::snmpEngineTime if available to + # fix device uptime if wrapped + if (defined $snmp->snmpEngineTime) { + $dev_uptime_wrapped = int( $snmp->snmpEngineTime * 100 / 2**32 ); + if ($dev_uptime_wrapped > 0) { + info sprintf ' [%s] interface - device uptime wrapped %d times - correcting', + $device->ip, $dev_uptime_wrapped; + $device->uptime( $dev_uptime + $dev_uptime_wrapped * 2**32 ); + } + } + + # build device interfaces suitable for DBIC + my %interfaces; + foreach my $entry (keys %$interfaces) { + my $port = $interfaces->{$entry}; + + if (not $port) { + debug sprintf ' [%s] interfaces - ignoring %s (no port mapping)', + $device->ip, $entry; + next; + } + + if (scalar grep {$port =~ m/^$_$/} @{setting('ignore_interfaces') || []}) { + debug sprintf + ' [%s] interfaces - ignoring %s (%s) (config:ignore_interfaces)', + $device->ip, $entry, $port; + next; + } + + if (exists $i_ignore->{$entry}) { + debug sprintf ' [%s] interfaces - ignoring %s (%s) (%s)', + $device->ip, $entry, $port, $i_type->{$entry}; + next; + } + + my $lc = $i_lastchange->{$entry} || 0; + if (not $dev_uptime_wrapped and $lc > $dev_uptime) { + info sprintf ' [%s] interfaces - device uptime wrapped (%s) - correcting', + $device->ip, $port; + $device->uptime( $dev_uptime + 2**32 ); + $dev_uptime_wrapped = 1; + } + + if ($device->is_column_changed('uptime') and $lc) { + if ($lc < $dev_uptime) { + # ambiguous: lastchange could be sysUptime before or after wrap + if ($dev_uptime > 30000 and $lc < 30000) { + # uptime wrap more than 5min ago but lastchange within 5min + # assume lastchange was directly after boot -> no action + } + else { + # uptime wrap less than 5min ago or lastchange > 5min ago + # to be on safe side, assume lastchange after counter wrap + debug sprintf + ' [%s] interfaces - correcting LastChange for %s, assuming sysUptime wrap', + $device->ip, $port; + $lc += $dev_uptime_wrapped * 2**32; + } + } + } + + $interfaces{$port} = { + port => $port, + descr => $i_descr->{$entry}, + up => $i_up->{$entry}, + up_admin => $i_up_admin->{$entry}, + mac => $i_mac->{$entry}, + speed => $i_speed->{$entry}, + mtu => $i_mtu->{$entry}, + name => Encode::decode('UTF-8', $i_name->{$entry}), + duplex => $i_duplex->{$entry}, + duplex_admin => $i_duplex_admin->{$entry}, + stp => $i_stp_state->{$entry}, + type => $i_type->{$entry}, + vlan => $i_vlan->{$entry}, + pvid => $i_vlan->{$entry}, + is_master => 'false', + slave_of => undef, + lastchange => $lc, + }; + } + + # must do this after building %interfaces so that we can set is_master + foreach my $sidx (keys %$agg_ports) { + my $slave = $interfaces->{$sidx} or next; + my $master = $interfaces->{ $agg_ports->{$sidx} } or next; + next unless exists $interfaces{$slave} and exists $interfaces{$master}; + + $interfaces{$slave}->{slave_of} = $master; + $interfaces{$master}->{is_master} = 'true'; + } + + schema('netdisco')->resultset('DevicePort')->txn_do_locked(sub { + my $gone = $device->ports->delete({keep_nodes => 1}); + debug sprintf ' [%s] interfaces - removed %d interfaces', + $device->ip, $gone; + $device->update_or_insert(undef, {for => 'update'}); + $device->ports->populate([values %interfaces]); + + return Status->noop(sprintf ' [%s] interfaces - added %d new interfaces', + $device->ip, scalar values %interfaces); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm b/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm new file mode 100644 index 00000000..02401389 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/VLANs.pm @@ -0,0 +1,95 @@ +package App::Netdisco::Worker::Plugin::Discover::VLANs; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $v_name = $snmp->v_name; + my $v_index = $snmp->v_index; + + # build device vlans suitable for DBIC + my %v_seen = (); + my @devicevlans; + foreach my $entry (keys %$v_name) { + my $vlan = $v_index->{$entry}; + next unless defined $vlan and $vlan; + ++$v_seen{$vlan}; + + push @devicevlans, { + vlan => $vlan, + description => $v_name->{$entry}, + last_discover => \'now()', + }; + } + + my $i_vlan = $snmp->i_vlan; + my $i_vlan_membership = $snmp->i_vlan_membership; + my $i_vlan_type = $snmp->i_vlan_type; + my $interfaces = $snmp->interfaces; + + # build device port vlans suitable for DBIC + my @portvlans = (); + foreach my $entry (keys %$i_vlan_membership) { + my %port_vseen = (); + my $port = $interfaces->{$entry}; + next unless defined $port; + + my $type = $i_vlan_type->{$entry}; + + foreach my $vlan (@{ $i_vlan_membership->{$entry} }) { + next unless defined $vlan and $vlan; + next if ++$port_vseen{$vlan} > 1; + + my $native = ((defined $i_vlan->{$entry}) and ($vlan eq $i_vlan->{$entry})) ? "t" : "f"; + push @portvlans, { + port => $port, + vlan => $vlan, + native => $native, + vlantype => $type, + last_discover => \'now()', + }; + + next if $v_seen{$vlan}; + + # also add an unnamed vlan to the device + push @devicevlans, { + vlan => $vlan, + description => (sprintf "VLAN %d", $vlan), + last_discover => \'now()', + }; + ++$v_seen{$vlan}; + } + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->vlans->delete; + debug sprintf ' [%s] vlans - removed %d device VLANs', + $device->ip, $gone; + $device->vlans->populate(\@devicevlans); + debug sprintf ' [%s] vlans - added %d new device VLANs', + $device->ip, scalar @devicevlans; + }); + + schema('netdisco')->txn_do(sub { + my $gone = $device->port_vlans->delete; + debug sprintf ' [%s] vlans - removed %d port VLANs', + $device->ip, $gone; + $device->port_vlans->populate(\@portvlans); + + return Status->noop(sprintf ' [%s] vlans - added %d new port VLANs', + $device->ip, scalar @portvlans); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm b/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm new file mode 100644 index 00000000..f19579ea --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/Wireless.pm @@ -0,0 +1,85 @@ +package App::Netdisco::Worker::Plugin::Discover::Wireless; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + return unless $device->in_storage; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("discover failed: could not SNMP connect to $device"); + + my $ssidlist = $snmp->i_ssidlist; + return unless scalar keys %$ssidlist; + + my $interfaces = $snmp->interfaces; + my $ssidbcast = $snmp->i_ssidbcast; + my $ssidmac = $snmp->i_ssidmac; + my $channel = $snmp->i_80211channel; + my $power = $snmp->dot11_cur_tx_pwr_mw; + + # build device ssid list suitable for DBIC + my @ssids; + foreach my $entry (keys %$ssidlist) { + (my $iid = $entry) =~ s/\.\d+$//; + my $port = $interfaces->{$iid}; + + if (not $port) { + debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', + $device->ip, $iid; + next; + } + + push @ssids, { + port => $port, + ssid => $ssidlist->{$entry}, + broadcast => $ssidbcast->{$entry}, + bssid => $ssidmac->{$entry}, + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->ssids->delete; + debug sprintf ' [%s] wireless - removed %d SSIDs', + $device->ip, $gone; + $device->ssids->populate(\@ssids); + debug sprintf ' [%s] wireless - added %d new SSIDs', + $device->ip, scalar @ssids; + }); + + # build device channel list suitable for DBIC + my @channels; + foreach my $entry (keys %$channel) { + my $port = $interfaces->{$entry}; + + if (not $port) { + debug sprintf ' [%s] wireless - ignoring %s (no port mapping)', + $device->ip, $entry; + next; + } + + push @channels, { + port => $port, + channel => $channel->{$entry}, + power => $power->{$entry}, + }; + } + + schema('netdisco')->txn_do(sub { + my $gone = $device->wireless_ports->delete; + debug sprintf ' [%s] wireless - removed %d wireless channels', + $device->ip, $gone; + $device->wireless_ports->populate(\@channels); + + return Status->noop(sprintf ' [%s] wireless - added %d new wireless channels', + $device->ip, scalar @channels); + }); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm b/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm new file mode 100644 index 00000000..0aec8326 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Discover/WithNodes.pm @@ -0,0 +1,38 @@ +package App::Netdisco::Worker::Plugin::Discover::WithNodes; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::JobQueue 'jq_insert'; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + my $device = $job->device; + + # if requested, and the device has not yet been + # arpniped/macsucked, queue those jobs now + return unless $device->in_storage + and $job->subaction and $job->subaction eq 'with-nodes'; + + if (!defined $device->last_macsuck and $device->has_layer(2)) { + jq_insert({ + device => $device->ip, + action => 'macsuck', + username => $job->username, + userip => $job->userip, + }); + } + + if (!defined $device->last_arpnip and $device->has_layer(3)) { + jq_insert({ + device => $device->ip, + action => 'arpnip', + username => $job->username, + userip => $job->userip, + }); + } +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm b/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm new file mode 100644 index 00000000..3bace148 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/DiscoverAll.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::DiscoverAll; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + + my %queued = map {$_ => 1} jq_queued('discover'); + my @devices = schema('netdisco')->resultset('Device')->search({ + -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], + })->get_column('ip')->all; + my @filtered_devices = grep {!exists $queued{$_}} @devices; + + jq_insert([ + map {{ + device => $_, + action => 'discover', + username => $job->username, + userip => $job->userip, + }} (@filtered_devices) + ]); + + return Status->done('Queued discover job for all devices'); +}); + +true; diff --git a/lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm b/lib/App/Netdisco/Worker/Plugin/Expire.pm similarity index 65% rename from lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm rename to lib/App/Netdisco/Worker/Plugin/Expire.pm index c1599084..28c93cc4 100644 --- a/lib/App/Netdisco/Backend/Worker/Poller/Expiry.pm +++ b/lib/App/Netdisco/Worker/Plugin/Expire.pm @@ -1,17 +1,14 @@ -package App::Netdisco::Backend::Worker::Poller::Expiry; +package App::Netdisco::Worker::Plugin::Expire; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; -use Dancer qw/:moose :syntax :script/; use Dancer::Plugin::DBIC 'schema'; - -use App::Netdisco::Backend::Util ':all'; use App::Netdisco::Util::Statistics 'update_stats'; -use Role::Tiny; -use namespace::clean; - -# expire devices and nodes according to config -sub expire { - my ($self, $job) = @_; +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; if (setting('expire_devices') and setting('expire_devices') > 0) { schema('netdisco')->txn_do(sub { @@ -54,25 +51,7 @@ sub expire { # now update stats update_stats(); - return job_done("Checked expiry and updated stats"); -} + return Status->done('Checked expiry and updated stats'); +}); -# expire nodes for a specific device -sub expirenodes { - my ($self, $job) = @_; - - return job_error('Missing device') unless $job->device; - - schema('netdisco')->txn_do(sub { - schema('netdisco')->resultset('Node')->search({ - switch => $job->device->ip, - ($job->port ? (port => $job->port) : ()), - })->delete( - ($job->extra ? () : ({ archive_nodes => 1 })) - ); - }); - - return job_done("Expired nodes for ". $job->device->ip); -} - -1; +true; diff --git a/lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm b/lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm new file mode 100644 index 00000000..01bb565c --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/ExpireNodes.pm @@ -0,0 +1,30 @@ +package App::Netdisco::Worker::Plugin::ExpireNodes; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + return Status->done('ExpireNodes is able to run'); +}); + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('Node')->search({ + switch => $job->device->ip, + ($job->port ? (port => $job->port) : ()), + })->delete( + ($job->extra ? () : ({ archive_nodes => 1 })) + ); + }); + + return Status->done('Expired nodes for '. $job->device->ip); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Graph.pm b/lib/App/Netdisco/Worker/Plugin/Graph.pm new file mode 100644 index 00000000..0629890f --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Graph.pm @@ -0,0 +1,14 @@ +package App::Netdisco::Worker::Plugin::Graph; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Graph (); + +register_worker({ phase => 'main' }, sub { + App::Netdisco::Util::Graph::graph(); + return Status->done('Generated graph data'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Location.pm b/lib/App/Netdisco/Worker/Plugin/Location.pm new file mode 100644 index 00000000..fbcf3c08 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Location.pm @@ -0,0 +1,43 @@ +package App::Netdisco::Worker::Plugin::Location; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + return Status->done('Location is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $data) = map {$job->$_} qw/device extra/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update location"); + + my $rv = $snmp->set_location($data); + + if (!defined $rv) { + return Status->error( + "failed to set location on $device: ". ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $new_data = ($snmp->location || ''); + if ($new_data ne $data) { + return Status->error("verify of location failed on $device: $new_data"); + } + + # update netdisco DB + $device->update({location => $data}); + + return Status->done("Updated location on $device to [$data]"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck.pm new file mode 100644 index 00000000..a051329e --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::Macsuck; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Device 'is_macsuckable_now'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + my $device = $job->device; + + return Status->error('macsuck failed: unable to interpret device param') + unless defined $device; + + return Status->error("macsuck skipped: $device not yet discovered") + unless $device->in_storage; + + return Status->defer("macsuck skipped: $device is pseudo-device") + if $device->is_pseudo; + + return Status->defer("arpnip skipped: $device has no layer 2 capability") + unless $device->has_layer(2); + + return Status->defer("macsuck deferred: $device is not macsuckable") + unless is_macsuckable_now($device); + + return Status->done('Macsuck is able to run.'); +}); + +true; diff --git a/lib/App/Netdisco/Core/Macsuck.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm similarity index 70% rename from lib/App/Netdisco/Core/Macsuck.pm rename to lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm index 8d9c20e6..488585c2 100644 --- a/lib/App/Netdisco/Core/Macsuck.pm +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm @@ -1,62 +1,25 @@ -package App::Netdisco::Core::Macsuck; +package App::Netdisco::Worker::Plugin::Macsuck::Nodes; -use Dancer qw/:syntax :script/; -use Dancer::Plugin::DBIC 'schema'; +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; +use App::Netdisco::Transport::SNMP (); use App::Netdisco::Util::Permission 'check_acl_no'; use App::Netdisco::Util::PortMAC 'get_port_macs'; use App::Netdisco::Util::Device 'match_devicetype'; use App::Netdisco::Util::Node 'check_mac'; use App::Netdisco::Util::SNMP 'snmp_comm_reindex'; +use Dancer::Plugin::DBIC 'schema'; use Time::HiRes 'gettimeofday'; +use Scope::Guard 'guard'; -use base 'Exporter'; -our @EXPORT = (); -our @EXPORT_OK = qw/ - do_macsuck - store_node - store_wireless_client_info -/; -our %EXPORT_TAGS = (all => \@EXPORT_OK); +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; -=head1 NAME - -App::Netdisco::Core::Macsuck - -=head1 DESCRIPTION - -Helper subroutines to support parts of the Netdisco application. - -There are no default exports, however the C<:all> tag will export all -subroutines. - -=head1 EXPORT_OK - -=head2 do_macsuck( $device, $snmp ) - -Given a Device database object, and a working SNMP connection, connect to a -device and discover the MAC addresses listed against each physical port -without a neighbor. - -If the device has VLANs, C will walk each VLAN to get the MAC -addresses from there. - -It will also gather wireless client information if C -configuration setting is enabled. - -=cut - -sub do_macsuck { - my ($device, $snmp) = @_; - - unless ($device->in_storage) { - debug sprintf - ' [%s] macsuck - skipping device not yet discovered', - $device->ip; - return; - } - - my $ip = $device->ip; + my $device = $job->device; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("macsuck failed: could not SNMP connect to $device"); # would be possible just to use now() on updated records, but by using this # same value for them all, we can if we want add a job at the end to @@ -64,9 +27,6 @@ sub do_macsuck { my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; my $total_nodes = 0; - # do this before we start messing with the snmp community string - store_wireless_client_info($device, $snmp, $now); - # cache the device ports to save hitting the database for many single rows my $device_ports = {map {($_->port => $_)} $device->ports(undef, {prefetch => {neighbor_alias => 'device'}})->all}; @@ -74,14 +34,18 @@ sub do_macsuck { my $interfaces = $snmp->interfaces; # get forwarding table data via basic snmp connection - my $fwtable = _walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports); + my $fwtable = walk_fwtable($device, $interfaces, $port_macs, $device_ports); # ...then per-vlan if supported - my @vlan_list = _get_vlan_list($device, $snmp); - foreach my $vlan (@vlan_list) { + my @vlan_list = get_vlan_list($device); + { + my $guard = guard { snmp_comm_reindex($snmp, $device, 0) }; + foreach my $vlan (@vlan_list) { snmp_comm_reindex($snmp, $device, $vlan); - my $pv_fwtable = _walk_fwtable($device, $snmp, $interfaces, $port_macs, $device_ports, $vlan); + my $pv_fwtable = + walk_fwtable($device, $interfaces, $port_macs, $device_ports, $vlan); $fwtable = {%$fwtable, %$pv_fwtable}; + } } # now it's time to call store_node for every node discovered @@ -91,7 +55,7 @@ sub do_macsuck { foreach my $vlan (reverse sort keys %$fwtable) { foreach my $port (keys %{ $fwtable->{$vlan} }) { debug sprintf ' [%s] macsuck - port %s vlan %s : %s nodes', - $ip, $port, $vlan, scalar keys %{ $fwtable->{$vlan}->{$port} }; + $device->ip, $port, $vlan, scalar keys %{ $fwtable->{$vlan}->{$port} }; # make sure this port is UP in netdisco (unless it's a lag master, # because we can still see nodes without a functioning aggregate) @@ -105,30 +69,31 @@ sub do_macsuck { for keys %{ $fwtable->{0} }; ++$total_nodes; - store_node($ip, $vlan, $port, $mac, $now); + store_node($device->ip, $vlan, $port, $mac, $now); } } } debug sprintf ' [%s] macsuck - %s updated forwarding table entries', - $ip, $total_nodes; + $device->ip, $total_nodes; # a use for $now ... need to archive dissapeared nodes my $archived = 0; if (setting('node_freshness')) { - $archived = schema('netdisco')->resultset('Node')->search({ - switch => $ip, - time_last => \[ "< ($now - ?::interval)", - setting('node_freshness') .' minutes' ], - })->update({ active => \'false' }); + $archived = schema('netdisco')->resultset('Node')->search({ + switch => $device->ip, + time_last => \[ "< ($now - ?::interval)", + setting('node_freshness') .' minutes' ], + })->update({ active => \'false' }); } debug sprintf ' [%s] macsuck - removed %d fwd table entries to archive', - $ip, $archived; + $device->ip, $archived; $device->update({last_macsuck => \$now}); -} + return Status->done("Ended macsuck for $device"); +}); =head2 store_node( $ip, $vlan, $port, $mac, $now? ) @@ -183,8 +148,11 @@ sub store_node { } # return a list of vlan numbers which are OK to macsuck on this device -sub _get_vlan_list { - my ($device, $snmp) = @_; +sub get_vlan_list { + my $device = shift; + + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return (); # already checked! return () unless $snmp->cisco_comm_indexing; @@ -295,11 +263,14 @@ sub _get_vlan_list { # walks the forwarding table (BRIDGE-MIB) for the device and returns a # table of node entries. -sub _walk_fwtable { - my ($device, $snmp, $interfaces, $port_macs, $device_ports, $comm_vlan) = @_; +sub walk_fwtable { + my ($device, $interfaces, $port_macs, $device_ports, $comm_vlan) = @_; my $skiplist = {}; # ports through which we can see another device my $cache = {}; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return $cache; # already checked! + my $fw_mac = $snmp->fw_mac; my $fw_port = $snmp->fw_port; my $fw_vlan = $snmp->qb_fw_vlan; @@ -368,8 +339,8 @@ sub _walk_fwtable { # do not expose MAC address tables via SNMP. relies on prefetched # neighbors otherwise it would kill the DB with device lookups. my $neigh_cannot_macsuck = eval { # can fail - check_acl_no(($device_port->neighbor || "0 but true"), 'macsuck_unsupported') || - match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') }; + check_acl_no(($device_port->neighbor || "0 but true"), 'macsuck_unsupported') || + match_devicetype($device_port->remote_type, 'macsuck_unsupported_type') }; if ($device_port->is_uplink) { if ($neigh_cannot_macsuck) { @@ -455,90 +426,4 @@ sub _walk_fwtable { return $cache; } -=head2 store_wireless_client_info( $device, $snmp, $now? ) - -Given a Device database object, and a working SNMP connection, connect to a -device and discover 802.11 related information for all connected wireless -clients. - -If the device doesn't support the 802.11 MIBs, then this will silently return. - -If the device does support the 802.11 MIBs but Netdisco's configuration -does not permit polling (C must be true) then a debug -message is logged and the subroutine returns. - -Otherwise, client information is gathered and stored to the database. - -Optionally, a third argument can be the literal string passed to the time_last -field of the database record. If not provided, it defauls to C. - -=cut - -sub store_wireless_client_info { - my ($device, $snmp, $now) = @_; - $now ||= 'now()'; - - my $cd11_txrate = $snmp->cd11_txrate; - return unless $cd11_txrate and scalar keys %$cd11_txrate; - - if (setting('store_wireless_clients')) { - debug sprintf ' [%s] macsuck - gathering wireless client info', - $device->ip; - } - else { - debug sprintf ' [%s] macsuck - dot11 info available but skipped due to config', - $device->ip; - return; - } - - my $cd11_rateset = $snmp->cd11_rateset(); - my $cd11_uptime = $snmp->cd11_uptime(); - my $cd11_sigstrength = $snmp->cd11_sigstrength(); - my $cd11_sigqual = $snmp->cd11_sigqual(); - my $cd11_mac = $snmp->cd11_mac(); - my $cd11_port = $snmp->cd11_port(); - my $cd11_rxpkt = $snmp->cd11_rxpkt(); - my $cd11_txpkt = $snmp->cd11_txpkt(); - my $cd11_rxbyte = $snmp->cd11_rxbyte(); - my $cd11_txbyte = $snmp->cd11_txbyte(); - my $cd11_ssid = $snmp->cd11_ssid(); - - while (my ($idx, $txrates) = each %$cd11_txrate) { - my $rates = $cd11_rateset->{$idx}; - my $mac = $cd11_mac->{$idx}; - next unless defined $mac; # avoid null entries - # there can be more rows in txrate than other tables - - my $txrate = defined $txrates->[$#$txrates] - ? int($txrates->[$#$txrates]) - : undef; - - my $maxrate = defined $rates->[$#$rates] - ? int($rates->[$#$rates]) - : undef; - - my $ssid = $cd11_ssid->{$idx} || 'unknown'; - - schema('netdisco')->txn_do(sub { - schema('netdisco')->resultset('NodeWireless') - ->search({ 'me.mac' => $mac, 'me.ssid' => $ssid }) - ->update_or_create({ - txrate => $txrate, - maxrate => $maxrate, - uptime => $cd11_uptime->{$idx}, - rxpkt => $cd11_rxpkt->{$idx}, - txpkt => $cd11_txpkt->{$idx}, - rxbyte => $cd11_rxbyte->{$idx}, - txbyte => $cd11_txbyte->{$idx}, - sigqual => $cd11_sigqual->{$idx}, - sigstrength => $cd11_sigstrength->{$idx}, - time_last => \$now, - }, { - order_by => [qw/mac ssid/], - for => 'update', - }); - }); - } -} - -1; +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm b/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm new file mode 100644 index 00000000..d11c5f5e --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Macsuck/WirelessNodes.pm @@ -0,0 +1,82 @@ +package App::Netdisco::Worker::Plugin::Macsuck::WirelessNodes; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP (); +use Dancer::Plugin::DBIC 'schema'; +use Time::HiRes 'gettimeofday'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + + my $device = $job->device; + my $snmp = App::Netdisco::Transport::SNMP->reader_for($device) + or return Status->defer("macsuck failed: could not SNMP connect to $device"); + + my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; + + my $cd11_txrate = $snmp->cd11_txrate; + return unless $cd11_txrate and scalar keys %$cd11_txrate; + + if (setting('store_wireless_clients')) { + debug sprintf ' [%s] macsuck - gathering wireless client info', + $device->ip; + } + else { + return Status->noop(sprintf ' [%s] macsuck - dot11 info available but skipped due to config', + $device->ip); + } + + my $cd11_rateset = $snmp->cd11_rateset(); + my $cd11_uptime = $snmp->cd11_uptime(); + my $cd11_sigstrength = $snmp->cd11_sigstrength(); + my $cd11_sigqual = $snmp->cd11_sigqual(); + my $cd11_mac = $snmp->cd11_mac(); + my $cd11_port = $snmp->cd11_port(); + my $cd11_rxpkt = $snmp->cd11_rxpkt(); + my $cd11_txpkt = $snmp->cd11_txpkt(); + my $cd11_rxbyte = $snmp->cd11_rxbyte(); + my $cd11_txbyte = $snmp->cd11_txbyte(); + my $cd11_ssid = $snmp->cd11_ssid(); + + while (my ($idx, $txrates) = each %$cd11_txrate) { + my $rates = $cd11_rateset->{$idx}; + my $mac = $cd11_mac->{$idx}; + next unless defined $mac; # avoid null entries + # there can be more rows in txrate than other tables + + my $txrate = defined $txrates->[$#$txrates] + ? int($txrates->[$#$txrates]) + : undef; + + my $maxrate = defined $rates->[$#$rates] + ? int($rates->[$#$rates]) + : undef; + + my $ssid = $cd11_ssid->{$idx} || 'unknown'; + + schema('netdisco')->txn_do(sub { + schema('netdisco')->resultset('NodeWireless') + ->search({ 'me.mac' => $mac, 'me.ssid' => $ssid }) + ->update_or_create({ + txrate => $txrate, + maxrate => $maxrate, + uptime => $cd11_uptime->{$idx}, + rxpkt => $cd11_rxpkt->{$idx}, + txpkt => $cd11_txpkt->{$idx}, + rxbyte => $cd11_rxbyte->{$idx}, + txbyte => $cd11_txbyte->{$idx}, + sigqual => $cd11_sigqual->{$idx}, + sigstrength => $cd11_sigstrength->{$idx}, + time_last => \$now, + }, { + order_by => [qw/mac ssid/], + for => 'update', + }); + }); + } +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Macwalk.pm b/lib/App/Netdisco/Worker/Plugin/Macwalk.pm new file mode 100644 index 00000000..01d49b85 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Macwalk.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::Macwalk; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + + my %queued = map {$_ => 1} jq_queued('macsuck'); + my @devices = schema('netdisco')->resultset('Device')->search({ + -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], + })->has_layer('2')->get_column('ip')->all; + my @filtered_devices = grep {!exists $queued{$_}} @devices; + + jq_insert([ + map {{ + device => $_, + action => 'macsuck', + username => $job->username, + userip => $job->userip, + }} (@filtered_devices) + ]); + + return Status->done('Queued macsuck job for all devices'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Monitor.pm b/lib/App/Netdisco/Worker/Plugin/Monitor.pm new file mode 100644 index 00000000..2cf41af0 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Monitor.pm @@ -0,0 +1,14 @@ +package App::Netdisco::Worker::Plugin::Monitor; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::NodeMonitor (); + +register_worker({ phase => 'main' }, sub { + App::Netdisco::Util::NodeMonitor::monitor(); + return Status->done('Generated monitor data'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Nbtstat.pm b/lib/App/Netdisco/Worker/Plugin/Nbtstat.pm new file mode 100644 index 00000000..cff91e92 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Nbtstat.pm @@ -0,0 +1,21 @@ +package App::Netdisco::Worker::Plugin::Nbtstat; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Device 'is_macsuckable'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + + return Status->error('nbtstat failed: unable to interpret device param') + unless defined $job->device; + + return Status->defer(sprintf 'nbtstat deferred: %s is not macsuckable', $job->device->ip) + unless is_macsuckable($job->device); + + return Status->done('Nbtstat is able to run.'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm b/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm new file mode 100644 index 00000000..7b12ebea --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Nbtstat/Core.pm @@ -0,0 +1,50 @@ +package App::Netdisco::Worker::Plugin::Nbtstat::Core; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Nbtstat qw/nbtstat_resolve_async store_nbt/; +use App::Netdisco::Util::Node 'is_nbtstatable'; +use Dancer::Plugin::DBIC 'schema'; +use Time::HiRes 'gettimeofday'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + my $host = $job->device->ip; + + # get list of nodes on device + my $interval = (setting('nbtstat_max_age') || 7) . ' day'; + my $rs = schema('netdisco')->resultset('NodeIp')->search({ + -bool => 'me.active', + -bool => 'nodes.active', + 'nodes.switch' => $host, + 'me.time_last' => \[ '>= now() - ?::interval', $interval ], + },{ + join => 'nodes', + columns => 'ip', + distinct => 1, + })->ip_version(4); + + my @ips = map {+{'ip' => $_}} + grep { is_nbtstatable( $_ ) } + $rs->get_column('ip')->all; + + # Unless we have IPs don't bother + if (scalar @ips) { + my $now = 'to_timestamp('. (join '.', gettimeofday) .')'; + my $resolved_nodes = nbtstat_resolve_async(\@ips); + + # update node_nbt with status entries + foreach my $result (@$resolved_nodes) { + if (defined $result->{'nbname'}) { + store_nbt($result, $now); + } + } + } + + return Status->done("Ended nbtstat for $host"); +}); + +true; + diff --git a/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm b/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm new file mode 100644 index 00000000..72d0c26f --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Nbtwalk.pm @@ -0,0 +1,31 @@ +package App::Netdisco::Worker::Plugin::Nbtwalk; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::JobQueue qw/jq_queued jq_insert/; +use Dancer::Plugin::DBIC 'schema'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + + my %queued = map {$_ => 1} jq_queued('nbtstat'); + my @devices = schema('netdisco')->resultset('Device')->search({ + -or => [ 'vendor' => undef, 'vendor' => { '!=' => 'netdisco' }], + })->has_layer('2')->get_column('ip')->all; + my @filtered_devices = grep {!exists $queued{$_}} @devices; + + jq_insert([ + map {{ + device => $_, + action => 'nbtstat', + username => $job->username, + userip => $job->userip, + }} (@filtered_devices) + ]); + + return Status->done('Queued nbtstat job for all devices'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/PortControl.pm b/lib/App/Netdisco/Worker/Plugin/PortControl.pm new file mode 100644 index 00000000..f767a874 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/PortControl.pm @@ -0,0 +1,76 @@ +package App::Netdisco::Worker::Plugin::PortControl; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; +use App::Netdisco::Util::Port ':all'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + return Status->error('Missing device (-d).') unless defined $job->device; + return Status->error('Missing port (-p).') unless defined $job->port; + return Status->error('Missing status (-e).') unless defined $job->subaction; + + vars->{'port'} = get_port($job->device, $job->port) + or return Status->error(sprintf "Unknown port name [%s] on device %s", + $job->port, $job->device); + + my $vlan_reconfig_check = vlan_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter vlan: $vlan_reconfig_check") + if $vlan_reconfig_check; + + return Status->done('PortControl is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $pn) = map {$job->$_} qw/device port/; + + # need to remove "-other" which appears for power/portcontrol + (my $sa = $job->subaction) =~ s/-\w+//; + $job->subaction($sa); + + if ($sa eq 'bounce') { + $job->subaction('down'); + my $status = _action($job); + return $status if $status->not_ok; + $job->subaction('up'); + } + + return _action($job); +}); + +sub _action { + my $job = shift; + my ($device, $pn, $data) = map {$job->$_} qw/device port subaction/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update up_admin"); + + my $iid = get_iid($snmp, vars->{'port'}) + or return Status->error("Failed to get port ID for [$pn] from $device"); + + my $rv = $snmp->set_i_up_admin($data, $iid); + + if (!defined $rv) { + return Status->error(sprintf 'Failed to set [%s] up_admin to [%s] on $device: %s', + $pn, $data, ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $state = ($snmp->i_up_admin($iid) || ''); + if (ref {} ne ref $state or $state->{$iid} ne $data) { + return Status->error("Verify of [$pn] up_admin failed on $device"); + } + + # update netdisco DB + vars->{'port'}->update({up_admin => $data}); + + return Status->done("Updated [$pn] up_admin on [$device] to [$data]"); +} + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/PortName.pm b/lib/App/Netdisco/Worker/Plugin/PortName.pm new file mode 100644 index 00000000..f5bfdb2a --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/PortName.pm @@ -0,0 +1,54 @@ +package App::Netdisco::Worker::Plugin::PortName; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; +use App::Netdisco::Util::Port ':all'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + return Status->error('Missing device (-d).') unless defined $job->device; + return Status->error('Missing port (-p).') unless defined $job->port; + return Status->error('Missing name (-e).') unless defined $job->subaction; + + vars->{'port'} = get_port($job->device, $job->port) + or return Status->error(sprintf "Unknown port name [%s] on device %s", + $job->port, $job->device); + + return Status->done('PortName is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $pn, $data) = map {$job->$_} qw/device port extra/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update alias"); + + my $iid = get_iid($snmp, vars->{'port'}) + or return Status->error("Failed to get port ID for [$pn] from $device"); + + my $rv = $snmp->set_i_alias($data, $iid); + + if (!defined $rv) { + return Status->error(sprintf 'Failed to set [%s] alias to [%s] on $device: %s', + $pn, $data, ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $state = ($snmp->i_alias($iid) || ''); + if (ref {} ne ref $state or $state->{$iid} ne $data) { + return Status->error("Verify of [$pn] alias failed on $device"); + } + + # update netdisco DB + vars->{'port'}->update({name => $data}); + + return Status->done("Updated [$pn] alias on [$device] to [$data]"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Power.pm b/lib/App/Netdisco/Worker/Plugin/Power.pm new file mode 100644 index 00000000..16cba738 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Power.pm @@ -0,0 +1,69 @@ +package App::Netdisco::Worker::Plugin::Power; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; +use App::Netdisco::Util::Port ':all'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + return Status->error('Missing device (-d).') unless defined $job->device; + return Status->error('Missing port (-p).') unless defined $job->port; + return Status->error('Missing status (-e).') unless defined $job->subaction; + + vars->{'port'} = get_port($job->device, $job->port) + or return Status->error(sprintf "Unknown port name [%s] on device %s", + $job->port, $job->device); + + my $vlan_reconfig_check = vlan_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter vlan: $vlan_reconfig_check") + if $vlan_reconfig_check; + + return Status->error("No PoE service on port [$pn] on device $device") + unless vars->{'port'}->power; + + return Status->done('Power is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $pn) = map {$job->$_} qw/device port/; + + # munge data + (my $data = $job->subaction) =~ s/-\w+//; # remove -other + $data = 'true' if $data =~ m/^(on|yes|up)$/; + $data = 'false' if $data =~ m/^(off|no|down)$/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update vlan"); + + my $powerid = get_powerid($snmp, vars->{'port'}) + or return Status->error("failed to get power ID for [$pn] from $device"); + + my $rv = $snmp->set_peth_port_admin($data, $powerid); + + if (!defined $rv) { + return Status->error(sprintf 'failed to set [%s] power to [%s] on [%s]: %s', + $pn, $data, $device, ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $state = ($snmp->peth_port_admin($powerid) || ''); + if (ref {} ne ref $state or $state->{$powerid} ne $data) { + return Status->error("Verify of [$pn] power failed on $device"); + } + + # update netdisco DB + vars->{'port'}->power->update({ + admin => $data, + status => ($data eq 'false' ? 'disabled' : 'searching'), + }); + + return Status->done("Updated [$pn] power status on $device to [$data]"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Psql.pm b/lib/App/Netdisco/Worker/Plugin/Psql.pm new file mode 100644 index 00000000..b47f3823 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Psql.pm @@ -0,0 +1,39 @@ +package App::Netdisco::Worker::Plugin::Psql; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; + + my $name = ($ENV{NETDISCO_DBNAME} || setting('database')->{name} || 'netdisco'); + my $host = setting('database')->{host}; + my $user = setting('database')->{user}; + my $pass = setting('database')->{pass}; + + my $portnum = undef; + if ($host and $host =~ m/([^;]+);port=(\d+)/) { + $host = $1; + $portnum = $2; + } + + $ENV{PGHOST} = $host if $host; + $ENV{PGPORT} = $portnum if defined $portnum; + $ENV{PGDATABASE} = $name; + $ENV{PGUSER} = $user; + $ENV{PGPASSWORD} = $pass; + $ENV{PGCLIENTENCODING} = 'UTF8'; + + if ($extra) { + system('psql', '-c', $extra); + } + else { + system('psql'); + } + + return Status->done('psql session closed.'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Renumber.pm b/lib/App/Netdisco/Worker/Plugin/Renumber.pm new file mode 100644 index 00000000..47d59d24 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Renumber.pm @@ -0,0 +1,39 @@ +package App::Netdisco::Worker::Plugin::Renumber; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use NetAddr::IP qw/:rfc3021 :lower/; +use App::Netdisco::Util::Device qw/get_device renumber_device/; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + + my $new_ip = NetAddr::IP->new($job->extra); + unless ($new_ip and $new_ip->addr ne '0.0.0.0') { + return Status->error("Bad host or IP: ".($job->extra || '0.0.0.0')); + } + + return Status->done('Renumber is able to run'); +}); + +register_worker({ phase => 'main' }, sub { + my ($job, $workerconf) = @_; + my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; + + my $old_ip = $device->ip; + my $new_ip = NetAddr::IP->new($extra); + + my $new_dev = get_device($new_ip->addr); + if ($new_dev and $new_dev->in_storage and ($new_dev->ip ne $device->ip)) { + return Status->error(sprintf "Already know new device as: %s.", $new_dev->ip); + } + + renumber_device($device, $new_ip); + return Status->done(sprintf 'Renumbered device %s to %s (%s).', + $device->ip, $new_ip, ($device->dns || '')); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Show.pm b/lib/App/Netdisco/Worker/Plugin/Show.pm new file mode 100644 index 00000000..c6f93b40 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Show.pm @@ -0,0 +1,37 @@ +package App::Netdisco::Worker::Plugin::Show; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use Data::Printer (); +use App::Netdisco::Transport::SNMP; + +register_worker({ phase => 'check' }, sub { + return Status->error('Missing device (-d).') + unless defined shift->device; + return Status->done('Show is able to run'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $port, $extra) = map {$job->$_} qw/device port extra/; + + $extra ||= 'interfaces'; my $class = undef; + ($class, $extra) = split(/::([^:]+)$/, $extra); + if ($class and $extra) { + $class = 'SNMP::Info::'.$class; + } + else { + $extra = $class; + undef $class; + } + + my $i = App::Netdisco::Transport::SNMP->reader_for($device, $class); + Data::Printer::p($i->$extra($port)); + + return Status->done( + sprintf "Showed %s response from %s", $extra, $device->ip); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Stats.pm b/lib/App/Netdisco/Worker/Plugin/Stats.pm new file mode 100644 index 00000000..7c0d6e99 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Stats.pm @@ -0,0 +1,14 @@ +package App::Netdisco::Worker::Plugin::Stats; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Statistics (); + +register_worker({ phase => 'main' }, sub { + App::Netdisco::Util::Statistics::update_stats(); + return Status->done('Updated statistics'); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Vlan.pm b/lib/App/Netdisco/Worker/Plugin/Vlan.pm new file mode 100644 index 00000000..89b6b478 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Vlan.pm @@ -0,0 +1,30 @@ +package App::Netdisco::Worker::Plugin::Vlan; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Util::Port ':all'; + +register_worker({ phase => 'check' }, sub { + my ($job, $workerconf) = @_; + my ($device, $pn, $data) = map {$job->$_} qw/device port extra/; + return Status->error('Missing device (-d).') if !defined $device; + return Status->error('Missing port (-p).') if !defined $pn; + return Status->error('Missing vlan (-e).') if !defined $data; + + vars->{'port'} = get_port($device, $pn) + or return Status->error("Unknown port name [$pn] on device $device"); + + my $port_reconfig_check = port_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter port: $port_reconfig_check") + if $port_reconfig_check; + + my $vlan_reconfig_check = vlan_reconfig_check(vars->{'port'}); + return Status->error("Cannot alter vlan: $vlan_reconfig_check") + if $vlan_reconfig_check; + + return Status->done("Vlan is able to run."); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm b/lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm new file mode 100644 index 00000000..fd574e36 --- /dev/null +++ b/lib/App/Netdisco/Worker/Plugin/Vlan/Core.pm @@ -0,0 +1,61 @@ +package App::Netdisco::Worker::Plugin::Vlan::Native; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +use App::Netdisco::Transport::SNMP; +use App::Netdisco::Util::Port ':all'; + +register_worker({ phase => 'early', driver => 'snmp' }, sub { + my ($job, $workerconf) = @_; + my ($device, $pn) = map {$job->$_} qw/device port/; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update vlan/pvid"); + + vars->{'iid'} = get_iid($snmp, vars->{'port'}) + or return Status->error("Failed to get port ID for [$pn] from $device"); + + return Status->noop("Vlan set can continue."); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return unless defined vars->{'iid'}; + _action($job, 'pvid'); + return _action($job, 'vlan'); +} + +sub _action { + my ($job, $slot) = @_; + my ($device, $pn, $data) = map {$job->$_} qw/device port extra/; + + my $getter = "i_${slot}"; + my $setter = "set_i_${slot}"; + + # snmp connect using rw community + my $snmp = App::Netdisco::Transport::SNMP->writer_for($device) + or return Status->defer("failed to connect to $device to update $slot"); + + my $rv = $snmp->$setter($data, vars->{'iid'}); + + if (!defined $rv) { + return Status->error(sprintf 'Failed to set [%s] %s to [%s] on $device: %s', + $pn, $slot, $data, ($snmp->error || '')); + } + + # confirm the set happened + $snmp->clear_cache; + my $state = ($snmp->$getter(vars->{'iid'}) || ''); + if (ref {} ne ref $state or $state->{ vars->{'iid'} } ne $data) { + return Status->error("Verify of [$pn] $slot failed on $device"); + } + + # update netdisco DB + vars->{'port'}->update({$slot => $data}); + + return Status->done("Updated [$pn] $slot on [$device] to [$data]"); +}); + +true; diff --git a/lib/App/Netdisco/Worker/Runner.pm b/lib/App/Netdisco/Worker/Runner.pm new file mode 100644 index 00000000..2a81ab0c --- /dev/null +++ b/lib/App/Netdisco/Worker/Runner.pm @@ -0,0 +1,85 @@ +package App::Netdisco::Worker::Runner; + +use Dancer qw/:moose :syntax/; +use App::Netdisco::Util::Device 'get_device'; +use App::Netdisco::Util::Permission qw/check_acl_no check_acl_only/; +use aliased 'App::Netdisco::Worker::Status'; + +use Try::Tiny; +use Module::Load (); +use Scope::Guard 'guard'; + +use Moo::Role; +use namespace::clean; + +with 'App::Netdisco::Worker::Loader'; +has 'job' => ( is => 'rw' ); + +# mixin code to run workers loaded via plugins +sub run { + my ($self, $job) = @_; + + die 'cannot reuse a worker' if $self->job; + die 'bad job to run()' + unless ref $job eq 'App::Netdisco::Backend::Job'; + + $self->job($job); + $job->device( get_device($job->device) ); + $self->load_workers(); + + # finalise job status when we exit + my $statusguard = guard { $job->finalise_status }; + + my @newuserconf = (); + my @userconf = @{ setting('device_auth') || [] }; + + # reduce device_auth by only/no + if (ref $job->device) { + foreach my $stanza (@userconf) { + my $no = (exists $stanza->{no} ? $stanza->{no} : undef); + my $only = (exists $stanza->{only} ? $stanza->{only} : undef); + + next if $no and check_acl_no($job->device, $no); + next if $only and not check_acl_only($job->device, $only); + + push @newuserconf, $stanza; + } + + # per-device action but no device creds available + return $job->add_status( Status->defer('deferred job with no device creds') ) + if 0 == scalar @newuserconf; + } + + # back up and restore device_auth + my $configguard = guard { set(device_auth => \@userconf) }; + set(device_auth => \@newuserconf); + + # run check phase and if there are workers then one MUST be successful + $self->run_workers('workers_check'); + return if not $job->check_passed; + + # run other phases + $self->run_workers("workers_${_}") for qw/early main user/; +} + +sub run_workers { + my $self = shift; + my $job = $self->job or die error 'no job in worker job slot'; + + my $set = shift + or return $job->add_status( Status->error('missing set param') ); + return unless ref [] eq ref $self->$set and 0 < scalar @{ $self->$set }; + + (my $phase = $set) =~ s/^workers_//; + $job->enter_phase($phase); + + foreach my $worker (@{ $self->$set }) { + try { $job->add_status( $worker->($job) ) } + catch { + debug "=> $_" if $_; + $job->add_status( Status->error($_) ); + }; + } +} + +true; diff --git a/lib/App/Netdisco/Worker/Status.pm b/lib/App/Netdisco/Worker/Status.pm new file mode 100644 index 00000000..b8234f75 --- /dev/null +++ b/lib/App/Netdisco/Worker/Status.pm @@ -0,0 +1,105 @@ +package App::Netdisco::Worker::Status; + +use strict; +use warnings; + +use Dancer qw/:moose :syntax !error/; + +use Moo; +use namespace::clean; + +has 'status' => ( + is => 'rw', + default => undef, +); + +has [qw/log phase/] => ( + is => 'rw', + default => '', +); + +=head1 INTRODUCTION + +The status can be: + +=over 4 + +=item * C + +At C phase, indicates the action may continue. At other phases, +indicates the worker has completed without error or has no work to do. + +=item * C + +Indicates that there is an error condition. Also used to quit a worker without +side effects that C and C have. + +=item * C + +Quits a worker. If the final recorded outcome for a device is C several +times in a row, then it may be skipped from further jobs. + +=back + +=head1 METHODS + +=head2 done, error, defer + +Shorthand for new() with setting param, accepts log as arg. + +=cut + +sub _make_new { + my ($self, $status, $log) = @_; + die unless $status; + my $new = (ref $self ? $self : $self->new()); + $new->log($log); + $new->status($status); + return $new; +} + +sub error { shift->_make_new('error', @_) } +sub done { shift->_make_new('done', @_) } +sub defer { shift->_make_new('defer', @_) } + +=head2 noop + +Simply logs a message at debug level if passed, and returns true. Used for +consistency with other Status class methods but really does nothing. + +=cut + +sub noop { + debug $_[1] if $_[1]; + return true; +} + +=head2 is_ok + +Returns true if status is C. + +=cut + +sub is_ok { return $_[0]->status eq 'done' } + +=head2 not_ok + +Returns true if status is C or C. + +=cut + +sub not_ok { return (not $_[0]->is_ok) } + +=head2 level + +A numeric constant for the status, to allow comparison. + +=cut + +sub level { + my $self = shift; + return (($self->status eq 'done') ? 3 + : ($self->status eq 'defer') ? 2 : 1); +} + +1; diff --git a/share/config.yml b/share/config.yml index 061320e8..f0ecb523 100644 --- a/share/config.yml +++ b/share/config.yml @@ -190,6 +190,11 @@ ignore_interfaces: - 'Virtual-Template\d+' - 'Virtual-Access\d+' - '(E|T)\d \d\/\d\/\d' + - 'InLoopback0' + - 'NULL\d' + - 'Register-Tunnel\d' + - 'Blade-Aggregation\d' + - 'M-GigabitEthernet\d/\d/\d' ignore_private_nets: false reverse_sysname: false phone_capabilities: @@ -253,6 +258,55 @@ job_prio: - nbtstat - expire +worker_plugins: + - 'Arpnip' + - 'Arpnip::Nodes' + - 'Arpnip::Subnets' + - 'Arpwalk' + - 'Contact' + - 'Delete' + - 'Discover' + - 'Discover::CanonicalIP' + - 'Discover::Entities' + - 'Discover::Neighbors' + - 'Discover::PortPower' + - 'Discover::Properties' + - 'Discover::VLANs' + - 'Discover::Wireless' + - 'Discover::WithNodes' + - 'DiscoverAll' + - 'Expire' + - 'ExpireNodes' + - 'Graph' + - 'Location' + - 'Macsuck' + - 'Macsuck::Nodes' + - 'Macsuck::WirelessNodes' + - 'Macwalk' + - 'Monitor' + - 'Nbtstat' + - 'Nbtstat::Core' + - 'Nbtwalk' + - 'PortControl' + - 'PortName' + - 'Power' + - 'Psql' + - 'Renumber' + - 'Show' + - 'Stats' + - 'Vlan' + - 'Vlan::Core' + +extra_worker_plugins: [] +# - Discover::ConfigBackup::CLI + +driver_priority: + restconf: 500 + netconf: 400 + eapi: 300 + cli: 200 + snmp: 100 + # --------------- # GraphViz Export # --------------- diff --git a/share/environments/deployment.yml b/share/environments/deployment.yml index fa41ac7e..aa126c3d 100644 --- a/share/environments/deployment.yml +++ b/share/environments/deployment.yml @@ -30,7 +30,7 @@ safe_password_store: true # SNMP community string(s) # ```````````````````````` -snmp_auth: +device_auth: - tag: 'default_v2_readonly' community: 'public' read: true diff --git a/xt/30-backend-workers.t b/xt/30-backend-workers.t new file mode 100644 index 00000000..7845277d --- /dev/null +++ b/xt/30-backend-workers.t @@ -0,0 +1,118 @@ +#!/usr/bin/env perl + +use strict; use warnings FATAL => 'all'; +use Test::More 0.88; + +use lib 'xt/lib'; + +use App::Netdisco; +use App::Netdisco::Backend::Job; + +use Try::Tiny; +use Dancer qw/:moose :script !pass/; + +# configure logging to force console output +my $CONFIG = config(); +$CONFIG->{logger} = 'console'; +$CONFIG->{log} = 'error'; +Dancer::Logger->init('console', $CONFIG); + +{ + package MyWorker; + use Moo; + with 'App::Netdisco::Worker::Runner'; +} + +# clear user device_auth and set our own +config->{'device_auth'} = [{driver => 'snmp'}, {driver => 'cli'}]; + +# TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +my $j1 = do_job('TestOne'); +is($j1->status, 'done', 'status is done'); +is($j1->log, 'OK: SNMP driver is successful.', + 'workers are run in decreasing priority until done'); + +my $j2 = do_job('TestTwo'); +is($j2->status, 'done', 'status is done'); +is($j2->log, 'OK: CLI driver is successful.', + 'lower priority driver not run if higher is successful'); + +config->{'device_auth'} = []; + +my $j3 = do_job('TestOne'); +is($j3->status, 'defer', 'status is defer'); +is($j3->log, 'deferred job with no device creds', + 'no matching config for workers'); + +config->{'device_auth'} = [{driver => 'snmp'}]; + +my $j4 = do_job('TestThree'); +is($j4->status, 'done', 'status is done'); +is($j4->log, 'OK: SNMP driver is successful.', + 'respect user config filtering the driver'); + +config->{'device_auth'} = [ + {driver => 'snmp', action => 'testthree'}, + {driver => 'cli', action => 'foo'}, +]; + +my $j5 = do_job('TestThree'); +is($j5->status, 'done', 'status is done'); +is($j5->log, 'OK: SNMP driver is successful.', + 'respect user config filtering the action'); + +config->{'device_auth'} = [ + {driver => 'snmp', action => 'testthree::_base_'}, + {driver => 'cli', action => 'testthree::foo'}, +]; + +my $j6 = do_job('TestThree'); +is($j6->status, 'done', 'status is done'); +is($j6->log, 'OK: SNMP driver is successful.', + 'respect user config filtering the namespace'); + +config->{'device_auth'} = [{driver => 'snmp'}]; + +my $j7 = do_job('TestFour'); +is($j7->status, 'done', 'status is done'); +is($j7->log, 'OK: custom driver is successful.', + 'override an action'); + +config->{'device_auth'} = [{driver => 'snmp'}]; + +my $j8 = do_job('TestFive'); +is($j8->status, 'done', 'status is done'); +is((scalar @{$j8->_statuslist}), 2, 'two workers ran'); +is($j8->_last_priority, 100, 'priority is for snmp'); +is($j8->log, 'OK: SNMP driver is successful.', + 'add to an action'); + +done_testing; + +# TESTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +sub do_job { + my $pkg = shift; + + # include local plugins + config->{'extra_worker_plugins'} = ["X::${pkg}"]; + + my $job = App::Netdisco::Backend::Job->new({ + job => 0, + device => '192.0.2.1', + action => lc($pkg), + }); + + try { + #info sprintf 'test: started at %s', scalar localtime; + MyWorker->new()->run($job); + #info sprintf 'test: %s: %s', $job->status, $job->log; + } + catch { + $job->status('error'); + $job->log("error running job: $_"); + }; + + return $job; +} diff --git a/xt/lib/App/NetdiscoX/Worker/Plugin/TestFive.pm b/xt/lib/App/NetdiscoX/Worker/Plugin/TestFive.pm new file mode 100644 index 00000000..6c89926a --- /dev/null +++ b/xt/lib/App/NetdiscoX/Worker/Plugin/TestFive.pm @@ -0,0 +1,17 @@ +package App::NetdiscoX::Worker::Plugin::TestFive; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +# info 'test: add to an action'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->error('NOT OK: additional worker at SNMP level.'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->done('OK: SNMP driver is successful.'); +}); + +true; diff --git a/xt/lib/App/NetdiscoX/Worker/Plugin/TestFour.pm b/xt/lib/App/NetdiscoX/Worker/Plugin/TestFour.pm new file mode 100644 index 00000000..2ea142fe --- /dev/null +++ b/xt/lib/App/NetdiscoX/Worker/Plugin/TestFour.pm @@ -0,0 +1,17 @@ +package App::NetdiscoX::Worker::Plugin::TestFour; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +# info 'test: override an action'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->done('NOT OK: SNMP driver should NOT be run.'); +}); + +register_worker({ phase => 'main', priority => 120 }, sub { + return Status->done('OK: custom driver is successful.'); +}); + +true; diff --git a/xt/lib/App/NetdiscoX/Worker/Plugin/TestOne.pm b/xt/lib/App/NetdiscoX/Worker/Plugin/TestOne.pm new file mode 100644 index 00000000..0dfeabfd --- /dev/null +++ b/xt/lib/App/NetdiscoX/Worker/Plugin/TestOne.pm @@ -0,0 +1,17 @@ +package App::NetdiscoX::Worker::Plugin::TestOne; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +# info 'test: workers are run in decreasing priority until done'; + +register_worker({ phase => 'main', driver => 'cli' }, sub { + return Status->noop('NOT OK: CLI driver is not the winner here.'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->done('OK: SNMP driver is successful.'); +}); + +true; diff --git a/xt/lib/App/NetdiscoX/Worker/Plugin/TestThree.pm b/xt/lib/App/NetdiscoX/Worker/Plugin/TestThree.pm new file mode 100644 index 00000000..0cda2cd0 --- /dev/null +++ b/xt/lib/App/NetdiscoX/Worker/Plugin/TestThree.pm @@ -0,0 +1,17 @@ +package App::NetdiscoX::Worker::Plugin::TestThree; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +# info 'test: respect user config filtering the driver, action and namespace'; + +register_worker({ phase => 'main', driver => 'cli' }, sub { + return Status->done('NOT OK: CLI driver should NOT be run.'); +}); + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->done('OK: SNMP driver is successful.'); +}); + +true; diff --git a/xt/lib/App/NetdiscoX/Worker/Plugin/TestTwo.pm b/xt/lib/App/NetdiscoX/Worker/Plugin/TestTwo.pm new file mode 100644 index 00000000..79be4133 --- /dev/null +++ b/xt/lib/App/NetdiscoX/Worker/Plugin/TestTwo.pm @@ -0,0 +1,17 @@ +package App::NetdiscoX::Worker::Plugin::TestTwo; + +use Dancer ':syntax'; +use App::Netdisco::Worker::Plugin; +use aliased 'App::Netdisco::Worker::Status'; + +# info 'test: lower priority driver not run if higher is successful'; + +register_worker({ phase => 'main', driver => 'snmp' }, sub { + return Status->done('NOT OK: SNMP driver should NOT be run.'); +}); + +register_worker({ phase => 'main', driver => 'cli' }, sub { + return Status->done('OK: CLI driver is successful.'); +}); + +true;