135/netdisco
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#1111 Support for OUI28/MA-M and OUI36/MA-S

Browse Source 
* new oui importer using IEEE csv for MA-L+M+S

* schema update for new vendor table

* change vendor to manufacturer because Device has a vendor field

* remove oui from manuf table, and update node oui after manuf update

* faster way to bulk update node oui

* switch from using oui table to manufacturer table for vendor lookup

* some other oui cleanup

* faster/scalable way to join a macaddr to manuf table

* remove device.oui support

* update node oui in bulk at end of macsuck run

* correct literal sql instead of bind

* more efficient to get oui base for each mac

* comment better the base lookup in macsuck
This commit is contained in:
Oliver Gorwits
2023-11-14 18:55:54 +00:00
committed by GitHub
parent 7766ce64d1
commit 534a9d9378
26 changed files with 427 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/App/Netdisco/DB.pm
View File

@@ -11,7 +11,7 @@ __PACKAGE__->load_namespaces(
);
our # try to hide from kwalitee
$VERSION = 84; # schema version used for upgrades, keep as integer
$VERSION = 85; # schema version used for upgrades, keep as integer
use Path::Class;
use File::ShareDir 'dist_dir';

8
lib/App/Netdisco/DB/Result/Device.pm
View File

@@ -384,14 +384,6 @@ sub renumber {
=head1 ADDITIONAL COLUMNS
=head2 oui
Returns the first half of the device MAC address.
=cut
sub oui { return substr( ((shift)->mac || ''), 0, 8 ) }
=head2 port_count
Returns the number of ports on this device. Enable this

22
lib/App/Netdisco/DB/Result/DevicePort.pm
View File

@@ -302,6 +302,8 @@ __PACKAGE__->many_to_many( vlans => 'port_vlans', 'vlan_entry' );
=head2 oui
DEPRECATED: USE MANUFACTURER INSTEAD
Returns the C<oui> table entry matching this Port. You can then join on this
relation and retrieve the Company name from the related table.
@@ -320,6 +322,26 @@ __PACKAGE__->belongs_to( oui => 'App::Netdisco::DB::Result::Oui',
{ join_type => 'LEFT' }
);
=head2 manufacturer
Returns the C<manufacturer> table entry matching this Port. You can then join on this
relation and retrieve the Company name from the related table.
The JOIN is of type LEFT, in case the Manufacturer table has not been populated.
=cut
__PACKAGE__->belongs_to( manufacturer => 'App::Netdisco::DB::Result::Manufacturer',
sub {
my $args = shift;
return {
"$args->{foreign_alias}.range" => { '@>' =>
\qq{('x' || lpad( translate( $args->{self_alias}.mac ::text, ':', ''), 16, '0')) ::bit(64) ::bigint} },
};
},
{ join_type => 'LEFT' }
);
=head1 ADDITIONAL METHODS
=head2 neighbor

29
lib/App/Netdisco/DB/Result/Manufacturer.pm Normal file
View File

@@ -0,0 +1,29 @@
use utf8;
package App::Netdisco::DB::Result::Manufacturer;
use strict;
use warnings;
use base 'App::Netdisco::DB::Result';
__PACKAGE__->table("manufacturer");
__PACKAGE__->add_columns(
"company",
{ data_type => "text", is_nullable => 1 },
"abbrev",
{ data_type => "text", is_nullable => 1 },
"base",
{ data_type => "text", is_nullable => 0 },
"bits",
{ data_type => "integer", is_nullable => 1 },
"first",
{ data_type => "macaddr", is_nullable => 1 },
"last",
{ data_type => "macaddr", is_nullable => 1 },
"range",
{ data_type => "int8range", is_nullable => 1 },
);
__PACKAGE__->set_primary_key("base");
1;

19
lib/App/Netdisco/DB/Result/Node.pm
View File

@@ -19,7 +19,7 @@ __PACKAGE__->add_columns(
"active",
{ data_type => "boolean", is_nullable => 1 },
"oui",
{ data_type => "varchar", is_nullable => 1, is_serializable => 0, size => 8 },
{ data_type => "varchar", is_nullable => 1, is_serializable => 0, size => 9 },
"time_first",
{
data_type => "timestamp",
@@ -149,6 +149,8 @@ __PACKAGE__->has_many( wireless => 'App::Netdisco::DB::Result::NodeWireless',
=head2 oui
DEPRECATED: USE MANUFACTURER INSTEAD
Returns the C<oui> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
@@ -159,6 +161,21 @@ The JOIN is of type LEFT, in case the OUI table has not been populated.
__PACKAGE__->belongs_to( oui => 'App::Netdisco::DB::Result::Oui', 'oui',
{ join_type => 'LEFT' } );
=head2 manufacturer
Returns the C<manufacturer> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
The JOIN is of type LEFT, in case the Manufacturer table has not been populated.
=cut
__PACKAGE__->belongs_to( manufacturer => 'App::Netdisco::DB::Result::Manufacturer', {
'foreign.base' => 'self.oui',
},
{ join_type => 'LEFT' }
);
=head1 ADDITIONAL COLUMNS
=head2 time_first_stamp

22
lib/App/Netdisco/DB/Result/NodeIp.pm
View File

@@ -41,6 +41,8 @@ __PACKAGE__->set_primary_key("mac", "ip");
=head2 oui
DEPRECATED: USE MANUFACTURER INSTEAD
Returns the C<oui> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
@@ -59,6 +61,26 @@ __PACKAGE__->belongs_to( oui => 'App::Netdisco::DB::Result::Oui',
{ join_type => 'LEFT' }
);
=head2 manufacturer
Returns the C<manufacturer> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
The JOIN is of type LEFT, in case the Manufacturer table has not been populated.
=cut
__PACKAGE__->belongs_to( manufacturer => 'App::Netdisco::DB::Result::Manufacturer',
sub {
my $args = shift;
return {
"$args->{foreign_alias}.range" => { '@>' =>
\qq{('x' || lpad( translate( $args->{self_alias}.mac ::text, ':', ''), 16, '0')) ::bit(64) ::bigint} },
};
},
{ join_type => 'LEFT' }
);
=head2 node_ips
Returns the set of all C<node_ip> entries which are associated together with

22
lib/App/Netdisco/DB/Result/NodeNbt.pm
View File

@@ -47,6 +47,8 @@ __PACKAGE__->set_primary_key("mac");
=head2 oui
DEPRECATED: USE MANUFACTURER INSTEAD
Returns the C<oui> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
@@ -65,6 +67,26 @@ __PACKAGE__->belongs_to( oui => 'App::Netdisco::DB::Result::Oui',
{ join_type => 'LEFT' }
);
=head2 manufacturer
Returns the C<manufacturer> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
The JOIN is of type LEFT, in case the Manufacturer table has not been populated.
=cut
__PACKAGE__->belongs_to( manufacturer => 'App::Netdisco::DB::Result::Manufacturer',
sub {
my $args = shift;
return {
"$args->{foreign_alias}.range" => { '@>' =>
\qq{('x' || lpad( translate( $args->{self_alias}.mac ::text, ':', ''), 16, '0')) ::bit(64) ::bigint} },
};
},
{ join_type => 'LEFT' }
);
=head2 nodes
Returns the set of C<node> entries associated with this IP. That is, all the

22
lib/App/Netdisco/DB/Result/NodeWireless.pm
View File

@@ -48,6 +48,8 @@ __PACKAGE__->set_primary_key("mac", "ssid");
=head2 oui
DEPRECATED: USE MANUFACTURER INSTEAD
Returns the C<oui> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
@@ -66,6 +68,26 @@ __PACKAGE__->belongs_to( oui => 'App::Netdisco::DB::Result::Oui',
{ join_type => 'LEFT' }
);
=head2 manufacturer
Returns the C<manufacturer> table entry matching this Node. You can then join on this
relation and retrieve the Company name from the related table.
The JOIN is of type LEFT, in case the Manufacturer table has not been populated.
=cut
__PACKAGE__->belongs_to( manufacturer => 'App::Netdisco::DB::Result::Manufacturer',
sub {
my $args = shift;
return {
"$args->{foreign_alias}.range" => { '@>' =>
\qq{('x' || lpad( translate( $args->{self_alias}.mac ::text, ':', ''), 16, '0')) ::bit(64) ::bigint} },
};
},
{ join_type => 'LEFT' }
);
=head2 node
Returns the C<node> table entry matching this wireless entry.

2
lib/App/Netdisco/DB/Result/Virtual/LastNode.pm
View File

@@ -25,7 +25,7 @@ __PACKAGE__->add_columns(
"active",
{ data_type => "boolean", is_nullable => 1 },
"oui",
{ data_type => "varchar", is_nullable => 1, size => 8 },
{ data_type => "varchar", is_nullable => 1, size => 9 },
"time_first",
{
data_type => "timestamp",

22
lib/App/Netdisco/DB/ResultSet/NodeIp.pm
View File

@@ -8,15 +8,15 @@ __PACKAGE__->load_components(qw/
+App::Netdisco::DB::ExplicitLocking
/);
my $order_by_time_last_and_join_oui = {
my $order_by_time_last_and_join_manufacturer = {
order_by => {'-desc' => 'time_last'},
'+columns' => [
'oui.company',
'oui.abbrev',
'manufacturer.company',
'manufacturer.abbrev',
{ time_first_stamp => \"to_char(time_first, 'YYYY-MM-DD HH24:MI')" },
{ time_last_stamp => \"to_char(time_last, 'YYYY-MM-DD HH24:MI')" },
],
join => 'oui'
join => 'manufacturer'
};
=head1 with_times
@@ -38,7 +38,7 @@ sub with_times {
my ($rs, $cond, $attrs) = @_;
return $rs
->search_rs({}, $order_by_time_last_and_join_oui)
->search_rs({}, $order_by_time_last_and_join_manufacturer)
->search($cond, $attrs);
}
@@ -69,7 +69,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back
@@ -93,7 +93,7 @@ sub search_by_ip {
$cond->{ip} = { $op => $ip };
return $rs
->search_rs({}, $order_by_time_last_and_join_oui)
->search_rs({}, $order_by_time_last_and_join_manufacturer)
->search($cond, $attrs);
}
@@ -137,7 +137,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back
@@ -183,7 +183,7 @@ sub search_by_dns {
delete $cond->{suffix};
return $rs
->search_rs({}, $order_by_time_last_and_join_oui)
->search_rs({}, $order_by_time_last_and_join_manufacturer)
->search($cond, $attrs);
}
@@ -212,7 +212,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back
@@ -227,7 +227,7 @@ sub search_by_mac {
if ref {} ne ref $cond or !exists $cond->{mac};
return $rs
->search_rs({}, $order_by_time_last_and_join_oui)
->search_rs({}, $order_by_time_last_and_join_manufacturer)
->search($cond, $attrs);
}

10
lib/App/Netdisco/DB/ResultSet/NodeNbt.pm
View File

@@ -11,11 +11,11 @@ __PACKAGE__->load_components(qw/
my $search_attr = {
order_by => {'-desc' => 'time_last'},
'+columns' => [
'oui.company',
'manufacturer.company',
{ time_first_stamp => \"to_char(time_first, 'YYYY-MM-DD HH24:MI')" },
{ time_last_stamp => \"to_char(time_last, 'YYYY-MM-DD HH24:MI')" },
],
join => 'oui'
join => 'manufacturer'
};
=head1 with_times
@@ -68,7 +68,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back
@@ -121,7 +121,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back
@@ -167,7 +167,7 @@ preformatted timestamps of the C<time_first> and C<time_last> fields.
=item *
A JOIN is performed on the OUI table and the OUI C<company> column prefetched.
A JOIN is performed on the Manufacturer table and the Manufacturer C<company> column prefetched.
=back

2
lib/App/Netdisco/Web/Plugin/Device/Ports.pm
View File

@@ -174,7 +174,7 @@ get '/ajax/content/device/ports' => require_login sub {
if param('n_netbios');
# retrieve vendor, if asked for
$set = $set->search({}, { prefetch => [{$nodes_name => 'oui'}] })
$set = $set->search({}, { prefetch => [{$nodes_name => 'manufacturer'}] })
if param('n_vendor');
}

8
lib/App/Netdisco/Web/Plugin/Report/IpInventory.pm
View File

@@ -88,11 +88,11 @@ get '/ajax/content/report/ipinventory' => require_login sub {
my $rs2 = schema(vars->{'tenant'})->resultset('NodeIp')->search(
undef,
{ join => ['oui', 'netbios'],
{ join => ['manufacturer', 'netbios'],
columns => [qw( ip mac time_first time_last dns active)],
'+select' => [ \'true AS node',
\qq/replace( date_trunc( 'minute', age( LOCALTIMESTAMP, me.time_last ) ) ::text, 'mon', 'month') AS age/,
'oui.company',
'manufacturer.company',
'netbios.nbname',
],
'+as' => [ 'node', 'age', 'vendor', 'nbname' ],
@@ -101,14 +101,14 @@ get '/ajax/content/report/ipinventory' => require_login sub {
my $rs3 = schema(vars->{'tenant'})->resultset('NodeNbt')->search(
undef,
{ join => ['oui'],
{ join => ['manufacturer'],
columns => [qw( ip mac time_first time_last )],
'+select' => [
\'null AS dns',
'active',
\'true AS node',
\qq/replace( date_trunc( 'minute', age( LOCALTIMESTAMP, time_last ) ) ::text, 'mon', 'month') AS age/,
'oui.company',
'manufacturer.company',
'nbname'
],
'+as' => [ 'dns', 'active', 'node', 'age', 'vendor', 'nbname' ],

6
lib/App/Netdisco/Web/Plugin/Report/NodeMultiIPs.pm
View File

@@ -19,15 +19,15 @@ get '/ajax/content/report/nodemultiips' => require_login sub {
my @results = schema(vars->{'tenant'})->resultset('Node')->search(
{},
{ select => [ 'mac', 'switch', 'port' ],
join => [qw/device ips oui/],
join => [qw/device ips manufacturer/],
'+columns' => [
{ 'dns' => 'device.dns' },
{ 'name' => 'device.name' },
{ 'ip_count' => { count => 'ips.ip' } },
{ 'vendor' => 'oui.company' }
{ 'vendor' => 'manufacturer.company' }
],
group_by => [
qw/ me.mac me.switch me.port device.dns device.name oui.company/
qw/ me.mac me.switch me.port device.dns device.name manufacturer.company/
],
having => \[ 'count(ips.ip) > ?', [ count => 1 ] ],
order_by => { -desc => [qw/count/] },

18
lib/App/Netdisco/Web/Plugin/Report/NodeVendor.pm
View File

@@ -46,9 +46,9 @@ get '/ajax/content/report/nodevendor/data' => require_login sub {
my $match = $vendor eq 'blank' ? undef : $vendor;
$rs = $rs->search( { 'oui.abbrev' => $match },
{ '+columns' => [qw/ device.dns device.name oui.abbrev oui.company /],
join => [qw/ oui device /],
$rs = $rs->search( { 'manufacturer.abbrev' => $match },
{ '+columns' => [qw/ device.dns device.name manufacturer.abbrev manufacturer.company /],
join => [qw/ manufacturer device /],
collapse => 1,
});
@@ -85,9 +85,9 @@ get '/ajax/content/report/nodevendor' => require_login sub {
my $match = $vendor eq 'blank' ? undef : $vendor;
$rs = $rs->search( { 'oui.abbrev' => $match },
{ '+columns' => [qw/ device.dns device.name oui.abbrev oui.company /],
join => [qw/ oui device /],
$rs = $rs->search( { 'manufacturer.abbrev' => $match },
{ '+columns' => [qw/ device.dns device.name manufacturer.abbrev manufacturer.company /],
join => [qw/ manufacturer device /],
collapse => 1,
});
@@ -101,10 +101,10 @@ get '/ajax/content/report/nodevendor' => require_login sub {
elsif ( !defined $vendor ) {
$rs = $rs->search(
{ },
{ join => 'oui',
select => [ 'oui.abbrev', 'oui.company', { count => {distinct => 'me.mac'}} ],
{ join => 'manufacturer',
select => [ 'manufacturer.abbrev', 'manufacturer.company', { count => {distinct => 'me.mac'}} ],
as => [qw/ abbrev vendor count /],
group_by => [qw/ oui.abbrev oui.company /]
group_by => [qw/ manufacturer.abbrev manufacturer.company /]
}
)->order_by( { -desc => 'count' } );

24
lib/App/Netdisco/Web/Plugin/Search/Node.pm
View File

@@ -148,36 +148,36 @@ get '/ajax/content/search/node' => require_login sub {
->search({-and => [@where_mac, @active, @times]}, {
order_by => {'-desc' => 'time_last'},
'+columns' => [
'oui.company',
'oui.abbrev',
'manufacturer.company',
'manufacturer.abbrev',
{ time_first_stamp => \"to_char(time_first, 'YYYY-MM-DD HH24:MI')" },
{ time_last_stamp => \"to_char(time_last, 'YYYY-MM-DD HH24:MI')" },
],
join => 'oui'
join => 'manufacturer'
});
my $netbios = schema(vars->{'tenant'})->resultset('NodeNbt')
->search({-and => [@where_mac, @active, @times]}, {
order_by => {'-desc' => 'time_last'},
'+columns' => [
'oui.company',
'oui.abbrev',
'manufacturer.company',
'manufacturer.abbrev',
{ time_first_stamp => \"to_char(time_first, 'YYYY-MM-DD HH24:MI')" },
{ time_last_stamp => \"to_char(time_last, 'YYYY-MM-DD HH24:MI')" },
],
join => 'oui'
join => 'manufacturer'
});
my $wireless = schema(vars->{'tenant'})->resultset('NodeWireless')->search(
{ -and => [@where_mac, @wifitimes] },
{ order_by => { '-desc' => 'time_last' },
'+columns' => [
'oui.company',
'oui.abbrev',
'manufacturer.company',
'manufacturer.abbrev',
{
time_last_stamp => \"to_char(time_last, 'YYYY-MM-DD HH24:MI')"
}],
join => 'oui'
join => 'manufacturer'
}
);
@@ -245,14 +245,14 @@ get '/ajax/content/search/node' => require_login sub {
}
# if the user selects Vendor search opt, then
# we'll try the OUI company name as a fallback
# we'll try the manufacturer company name as a fallback
if (param('show_vendor') and not $have_rows) {
$set = schema(vars->{'tenant'})->resultset('NodeIp')
->with_times
->search(
{'oui.company' => { -ilike => ''.sql_match($node)}, @times},
{'prefetch' => 'oui'},
{'manufacturer.company' => { -ilike => ''.sql_match($node)}, @times},
{'prefetch' => 'manufacturer'},
);
++$have_rows if $set->has_rows;
}

8
lib/App/Netdisco/Web/Report.pm
View File

@@ -30,10 +30,10 @@ get '/report/*' => require_login sub {
$vendor_list = [
schema(vars->{'tenant'})->resultset('Node')->search(
{},
{ join => 'oui',
columns => ['oui.abbrev'],
order_by => 'oui.abbrev',
group_by => 'oui.abbrev',
{ join => 'manufacturer',
columns => ['manufacturer.abbrev'],
order_by => 'manufacturer.abbrev',
group_by => 'manufacturer.abbrev',
}
)->get_column('abbrev')->all
];

9
lib/App/Netdisco/Worker/Plugin/Macsuck/Nodes.pm
View File

@@ -235,6 +235,13 @@ sub store_node {
$now ||= 'LOCALTIMESTAMP';
$vlan ||= 0;
# ideally we just store the first 36 bits of the mac in the oui field
# and then no need for this query. haven't yet worked out the SQL for that.
my $oui = schema('netdisco')->resultset('Manufacturer')
->search({ range => { '@>' =>
\[q{('x' || lpad( translate( ? ::text, ':', ''), 16, '0')) ::bit(64) ::bigint}, $mac]} },
{ rows => 1, columns => 'base' })->first;
schema('netdisco')->txn_do(sub {
my $nodes = schema('netdisco')->resultset('Node');
@@ -257,7 +264,7 @@ sub store_node {
vlan => $vlan,
mac => $mac,
active => \'true',
oui => substr($mac,0,8),
oui => ($oui ? $oui->base : undef),
time_last => \$now,
(($old != 0) ? (time_recent => \$now) : ()),
},