Skip to content

Commit

Permalink
Refactor Syslog Handling
Browse files Browse the repository at this point in the history
remove global usage
namespaced code
New entry point to avoid legacy code initialization: lnms handle:syslog
  • Loading branch information
murrant committed Apr 26, 2023
1 parent 121c8ff commit 0a1241a
Show file tree
Hide file tree
Showing 8 changed files with 351 additions and 246 deletions.
56 changes: 52 additions & 4 deletions LibreNMS/Cache/Device.php
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,25 @@ public function get(int|string|null $device): \App\Models\Device
*/
public function getByHostname($hostname): \App\Models\Device
{
$device_id = array_column($this->devices, 'device_id', 'hostname')[$hostname] ?? 0;
return $this->getByField('hostname', $hostname);
}

/**
* Get device by any device field or a number of fields
* Slower than by device_id, but attempts to prevent an sql query
*/
public function getByField(string|array $fields, mixed $value): \App\Models\Device
{
$fields = (array)$fields;

return $this->devices[$device_id] ?? $this->load($hostname, 'hostname');
foreach ($fields as $field) {
$device_id = array_column($this->devices, 'device_id', $field)[$value] ?? 0;
if ($device_id) {
break;
}
}

return $this->devices[$device_id] ?? $this->load($value, $fields);

Check failure on line 110 in LibreNMS/Cache/Device.php

View workflow job for this annotation

GitHub Actions / PHP Static Analysis (8.1)

Variable $device_id might not be defined.
}

/**
Expand Down Expand Up @@ -123,9 +139,21 @@ public function has(int $device_id): bool
return isset($this->devices[$device_id]);
}

private function load(mixed $value, string $field = 'device_id'): \App\Models\Device
private function load(mixed $value, string|array $field = ['device_id']): \App\Models\Device
{
$device = \App\Models\Device::query()->where($field, $value)->first();
$query = \App\Models\Device::query();
foreach ((array) $field as $column) {
if ($column == 'ip') {
$value = inet_pton($value); // convert IP to binary for query
if ($value === false) {
continue; // not an IP, skip the ip field
}
}

$query->orWhere($column, $value);
}

$device = $query->first();

if (! $device) {
return new \App\Models\Device;
Expand All @@ -135,4 +163,24 @@ private function load(mixed $value, string $field = 'device_id'): \App\Models\De

return $device;
}

/**
* Insert a fake device into the cache to avoid database lookups
* Will not work with relationships unless they are pre-populated (and not using a relationship based query)
*/
public function fake(\App\Models\Device $device): \App\Models\Device
{
if (empty($device->device_id)) {
// find a free device_id
$device->device_id = 1;
while (isset($this->devices[$device->device_id])) {
$device->device_id++;
}
}

$device->exists = true; // fake that device is saved to database
$this->devices[$device->device_id] = $device;

return $device;
}
}
211 changes: 211 additions & 0 deletions LibreNMS/Syslog.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
<?php
/**
* Syslog.php
*
* -Description-
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* @link https://www.librenms.org
*
* @copyright 2023 Tony Murray
* @author Tony Murray <murraytony@gmail.com>
*/

namespace LibreNMS;

use App\Facades\DeviceCache;
use App\Models\Device;
use App\Models\Ipv4Address;
use Illuminate\Support\Arr;

class Syslog
{
private array $host_map;
private array $filter;
private array $host_xlate;
private bool $hooks_enabled;

public function __construct() {
$this->host_map = [];
$this->filter = Arr::wrap(Config::get('syslog_filter'));
$this->host_xlate = Arr::wrap(Config::get('syslog_xlate'));
$this->hooks_enabled = (bool) Config::get('enable_syslog_hooks', false);
}

public function process(array $entry, bool $update = true) {
foreach ($this->filter as $bi) {
if (isset($entry['msg']) && str_contains($entry['msg'], $bi)) {
return $entry;
}
}

$entry['host'] = preg_replace('/^::ffff:/', '', $entry['host']); // remove ipv6 socket prefix for ipv4
$entry['host'] = $this->host_xlate[$entry['host']] ?? $entry['host']; // translate host based on config

$device = $this->findHost($entry['host']);

if ($device->exists) {
$entry['device_id'] = $device->device_id;
$this->executeHooks($device, $entry);
$entry = $this->handleOsSpecificTweaks($device, $entry);

// handle common case fields were msg is missing
if (! isset($entry['program'])) {
$entry['program'] = $entry['msg'];
unset($entry['msg']);
}

if (isset($entry['program'])) {
$entry['program'] = strtoupper($entry['program']);
}

$entry = array_map(fn($value) => is_string($value) ? trim($value) : $value, $entry);

if ($update) {
\App\Models\Syslog::create($entry);
}
}

return $entry;
}

public function findHost(string $host): Device
{
if (empty($this->host_map[$host])) {
// try hostname
$device = DeviceCache::getByField(['hostname', 'sysName', 'ip'], $host);

if (! $device->exists) {
// If failed, try by IPs on interfaces
$device_id = Ipv4Address::query()->leftJoin('ports', 'ipv4_addresses.port_id', '=', 'ports.port_id')

Check failure on line 92 in LibreNMS/Syslog.php

View workflow job for this annotation

GitHub Actions / PHP Static Analysis (8.1)

Property 'device_id' does not exist in App\Models\Ipv4Address model.
->where('ipv4_address', $host)->value('device_id');
$device = DeviceCache::get($device_id);
}

$this->host_map[$host] = $device->device_id; // save to map
}

return DeviceCache::get($this->host_map[$host]);
}

private function executeHooks(Device $device, array $entry): void
{
if ($this->hooks_enabled && is_array($hooks = Config::getOsSetting($device->os, 'syslog_hook'))) {
foreach ($hooks as $hook) {
$syslogprogmsg = $entry['program'] . ': ' . $entry['msg'];
if ((isset($hook['script'])) && (isset($hook['regex'])) && preg_match($hook['regex'], $syslogprogmsg)) {
shell_exec(escapeshellcmd($hook['script']) . ' ' . escapeshellarg($device->hostname) . ' ' . escapeshellarg($device->os) . ' ' . escapeshellarg($syslogprogmsg) . ' >/dev/null 2>&1 &');
}
}
}
}

private function handleOsSpecificTweaks(Device $device, array $entry): array
{
// ios like
if (in_array($device->os, ['ios', 'iosxe', 'catos'])) {
// multipart message
if (strpos($entry['msg'], ':') !== false) {
$timestamp_prefix = '([\*\.]?[A-Z][a-z]{2} \d\d? \d\d:\d\d:\d\d(.\d\d\d)?( [A-Z]{3})?: )?';
$program_match = '(?<program>%?[A-Za-z\d\-_]+(:[A-Z]* %[A-Z\d\-_]+)?)';
$message_match = '(?<msg>.*)';
if (preg_match('/^' . $timestamp_prefix . $program_match . ': ?' . $message_match . '/', $entry['msg'], $matches)) {
$entry['program'] = $matches['program'];
$entry['msg'] = $matches['msg'];
}
} else {
// if this looks like a program (no groups of 2 or more lowercase letters), move it to program
if (! preg_match('/[(a-z)]{2,}/', $entry['msg'])) {
$entry['program'] = $entry['msg'];
unset($entry['msg']);
}
}

return $entry;
}

if ($device->os == 'linux') {
// Cisco WAP200 and similar
if ($device->version == 'Point') {
if (preg_match('#Log: \[(?P<program>.*)\] - (?P<msg>.*)#', $entry['msg'], $matches)) {
$entry['msg'] = $matches['msg'];
$entry['program'] = $matches['program'];
}

return $entry;
}

// regular linux
// pam_krb5(sshd:auth): authentication failure; logname=root uid=0 euid=0 tty=ssh ruser= rhost=123.213.132.231
// pam_krb5[sshd:auth]: authentication failure; logname=root uid=0 euid=0 tty=ssh ruser= rhost=123.213.132.231
if (empty($entry['program']) && isset($entry['msg']) && preg_match('#^(?P<program>([^(:]+\([^)]+\)|[^\[:]+\[[^\]]+\])) ?: ?(?P<msg>.*)$#', $entry['msg'], $matches)) {
$entry['msg'] = $matches['msg'];
$entry['program'] = $matches['program'];
} elseif (empty($entry['program']) && ! empty($entry['facility'])) {
// SYSLOG CONNECTION BROKEN; FD='6', SERVER='AF_INET(123.213.132.231:514)', time_reopen='60'
// pam_krb5: authentication failure; logname=root uid=0 euid=0 tty=ssh ruser= rhost=123.213.132.231
// Disabled because broke this:
// diskio.c: don't know how to handle 10 request
// elseif($pos = strpos($entry['msg'], ';') or $pos = strpos($entry['msg'], ':')) {
// $entry['program'] = substr($entry['msg'], 0, $pos);
// $entry['msg'] = substr($entry['msg'], $pos+1);
// }
// fallback, better than nothing...
$entry['program'] = $entry['facility'];
}

return $entry;
}

// HP ProCurve
if ($device->os == 'procurve') {
if (preg_match('/^(?P<program>[A-Za-z]+): {2}(?P<msg>.*)/', $entry['msg'], $matches)) {
$entry['msg'] = $matches['msg'] . ' [' . $entry['program'] . ']';
$entry['program'] = $matches['program'];
}

return $entry;
}

// Zwwall sends messages without all the fields, so the offset is wrong
if ($device->os == 'zywall') {
$msg = preg_replace('/" /', '";', stripslashes($entry['program'] . ':' . $entry['msg']));
$msg = str_getcsv($msg, ';');
$entry['program'] = null;
foreach ($msg as $param) {
[$var, $val] = explode('=', $param);
if ($var == 'cat') {
$entry['program'] = str_replace('"', '', $val);
}
}
$entry['msg'] = join(' ', $msg);

return $entry;
}

return $entry;
}

public function getMap(): array{
return $this->host_map;
}

public function reset()
{
Config::reload();
DeviceCache::flush();
$this->__construct();
}
}
10 changes: 10 additions & 0 deletions app/Models/Syslog.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,14 @@ class Syslog extends DeviceRelatedModel
protected $table = 'syslog';
protected $primaryKey = 'seq';
public $timestamps = false;
protected $fillable = [
'device_id',
'program',
'facility',
'priority',
'level',
'tag',
'msg',
'timestamp',
];
}
10 changes: 5 additions & 5 deletions doc/Extensions/Syslog.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ source s_net {
};

destination d_librenms {
program("/opt/librenms/syslog.php" template ("$HOST||$FACILITY||$PRIORITY||$LEVEL||$TAG||$R_YEAR-$R_MONTH-$R_DAY $R_HOUR:$R_MIN:$R_SEC||$MSG||$PROGRAM\n") template-escape(yes));
program("/opt/librenms/lnms handle:syslog" template ("$HOST||$FACILITY||$PRIORITY||$LEVEL||$TAG||$R_YEAR-$R_MONTH-$R_DAY $R_HOUR:$R_MIN:$R_SEC||$MSG||$PROGRAM\n") template-escape(yes));
};

log {
Expand Down Expand Up @@ -103,7 +103,7 @@ Create a file called `/etc/rsyslog.d/30-librenms.conf`and add the following depe
type="string"
string= "%fromhost%||%syslogfacility%||%syslogpriority%||%syslogseverity%||%syslogtag%||%$year%-%$month%-%$day% %timegenerated:8:25%||%msg%||%programname%\n")
action(type="omprog"
binary="/opt/librenms/syslog.php"
binary="/opt/librenms/lnms handle:syslog"
template="librenms")

& stop
Expand All @@ -116,7 +116,7 @@ Create a file called `/etc/rsyslog.d/30-librenms.conf`and add the following depe

$template librenms,"%fromhost%||%syslogfacility%||%syslogpriority%||%syslogseverity%||%syslogtag%||%$year%-%$month%-%$day% %timegenerated:8:25%||%msg%||%programname%\n"

*.* action(type="omprog" binary="/opt/librenms/syslog.php" template="librenms")
*.* action(type="omprog" binary="/opt/librenms/lnms handle:syslog" template="librenms")

& stop

Expand All @@ -128,7 +128,7 @@ Create a file called `/etc/rsyslog.d/30-librenms.conf`and add the following depe
$ModLoad omprog
$template librenms,"%FROMHOST%||%syslogfacility-text%||%syslogpriority-text%||%syslogseverity%||%syslogtag%||%$YEAR%-%$MONTH%-%$DAY% %timegenerated:8:25%||%msg%||%programname%\n"

$ActionOMProgBinary /opt/librenms/syslog.php
$ActionOMProgBinary /opt/librenms/lnms handle:syslog
*.* :omprog:;librenms
```

Expand Down Expand Up @@ -164,7 +164,7 @@ syslog {
output {
exec {
command => "echo `echo %{host},,,,%{facility},,,,%{priority},,,,%{severity},,,,%{facility_label},,,,``date --date='%{timestamp}' '+%Y-%m-%d %H:%M:%S'``echo ',,,,%{message}'``echo ,,,,%{program} | sed 's/\x25\x7b\x70\x72\x6f\x67\x72\x61\x6d\x7d/%{facility_label}/'` | sed 's/,,,,/||/g' | /opt/librenms/syslog.php &"
command => "echo `echo %{host},,,,%{facility},,,,%{priority},,,,%{severity},,,,%{facility_label},,,,``date --date='%{timestamp}' '+%Y-%m-%d %H:%M:%S'``echo ',,,,%{message}'``echo ,,,,%{program} | sed 's/\x25\x7b\x70\x72\x6f\x67\x72\x61\x6d\x7d/%{facility_label}/'` | sed 's/,,,,/||/g' | /opt/librenms/lnms handle:syslog &"
}
elasticsearch {
hosts => ["10.10.10.10:9200"]
Expand Down
3 changes: 2 additions & 1 deletion includes/html/api_functions.inc.php
Original file line number Diff line number Diff line change
Expand Up @@ -2937,9 +2937,10 @@ function post_syslogsink(Illuminate\Http\Request $request)
}

$logs = array_is_list($json) ? $json : [$json];
$syslog = new \LibreNMS\Syslog;

foreach ($logs as $entry) {
process_syslog($entry, 1);
$syslog->process($entry);
}

return api_success_noresult(200, 'Syslog received: ' . count($logs));
Expand Down
Loading

0 comments on commit 0a1241a

Please sign in to comment.