Adjust logs for containerized and non-containerized services

addons/upgrade/to-12.0-rename-log-files.pl

@@ -0,0 +1,91 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+to-12.0-rename-log-files.pl
+
+=head1 DESCRIPTION
+
+Rename log files defined in logs= in syslog.conf
+
+=cut
+
+use strict;
+use warnings;
+use lib qw(/usr/local/pf/lib /usr/local/pf/lib_perl/lib/perl5);
+use pf::util;
+use pf::IniFiles;
+use pf::constants::config;
+use pf::file_paths qw(
+    $conf_dir
+    $syslog_config_file
+);
+use List::MoreUtils qw(any);
+
+run_as_pf();
+
+my %remap = (
+    'mariadb_error.log'           => 'mariadb.log',
+    'httpd.aaa.access'            => 'httpd.apache',
+    'httpd.aaa.error'             => 'httpd.apache',
+    'httpd.collector.log'         => 'httpd.apache',
+    'httpd.collector.error'       => 'httpd.apache',
+    'httpd.portal.error'          => 'httpd.apache',
+    'httpd.portal.access'         => 'httpd.apache',
+    'httpd.portal.catalyst'       => 'httpd.apache',
+    'httpd.proxy.error'           => 'httpd.apache',
+    'httpd.proxy.access'          => 'httpd.apache',
+    'httpd.webservices.error'     => 'httpd.apache',
+    'httpd.webservices.access'    => 'httpd.apache',
+    'httpd.api-frontend.access'   => 'httpd.apache',
+);
+
+my $ini = pf::IniFiles->new( -file => $syslog_config_file, -allowempty => 1);
+my $i = 0;
+
+for my $section ($ini->Sections()) {
+    if (my $logs = $ini->val($section, 'logs')) {
+        $logs = [ split(/,/, $logs) ];
+
+        if(any {exists $remap{$_}} @$logs) {
+            print "Renaming log files in section $section in file $syslog_config_file\n";
+            $logs = [ map { exists($remap{$_}) ? $remap{$_} : $_ } @$logs ];
+            $ini->setval($section, 'logs', join(',', @$logs));
+	    $i |= 1;
+        }
+    }
+}
+if ($i) {
+    $ini->RewriteConfig();
+    print "All done\n";
+} else {
+    print "Nothing to be done\n";
+}
+
+=head1 AUTHOR
+
+Inverse inc. <info@inverse.ca>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2005-2022 Inverse inc.
+
+=head1 LICENSE
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+USA.
+
+=cut
+

conf/monitoring/statsd.d/packetfence.conf.example

@@ -163,15 +163,15 @@
     type = line
     dimension = pattern 'source.packetfence.radius_log.*' '' last 1 1
 
-[logs.mariadb_error_log]
-    name = packetfence.mariadb_error_log
-    title = mariadb_error.log file events
+[logs.mariadb_log]
+    name = packetfence.mariadb_log
+    title = mariadb.log file events
     family = packetfence
     context = chart.context
     units = Events
     priority = 91000
     type = line
-    dimension = pattern 'source.packetfence.mariadb_error_log.*' '' last 1 1
+    dimension = pattern 'source.packetfence.mariadb_log.*' '' last 1 1
 
 [logs.pfcron_log]
     name = packetfence.pfcron_log

conf/stats.conf.defaults

@@ -239,11 +239,11 @@ match=ERROR
 statsd_ns=source.packetfence.radius_acct_log.error
 management=false
 
-[metric 'Events in mariadb_error.log']
+[metric 'Events in mariadb.log']
 type=tail_file
-file=/usr/local/pf/logs/mariadb_error.log
+file=/usr/local/pf/logs/mariadb.log
 match=DESYNCED
-statsd_ns=source.packetfence.mariadb_error_log.desynced
+statsd_ns=source.packetfence.mariadb_log.desynced
 management=false
 
 [metric 'Events in pfcron.log']

containers/daemon.json

@@ -1,4 +1,5 @@
 {
+  "log-driver": "none",
   "default-address-pools": [
     {
       "base": "100.64.0.0/10",

debian/packetfence-redis-cache.postinst

@@ -22,9 +22,7 @@ DIST=$(lsb_release -c -s)
 case "$1" in
     configure)
 	export PACKETFENCE=/usr/local/pf
-	chmod 2775 -R $PACKETFENCE/logs
 	chmod 2775 -R $PACKETFENCE/var
-	chown pf:pf $PACKETFENCE/logs
 	chown pf:pf $PACKETFENCE/var
         if [ ${DIST} = "wheezy" ] || [ ${DIST} = "precise" ]; then
             update-rc.d packetfence-redis-cache defaults 60 || exit 0

debian/packetfence.postinst

@@ -32,7 +32,6 @@ case "$1" in
         chmod 2775 -R $PACKETFENCE/var
         find $PACKETFENCE/var/conf -type f -exec chmod 664 '{}' \;
         find $PACKETFENCE/var/conf -type d -exec chmod 775 '{}' \;
-        chmod 2775 -R $PACKETFENCE/logs
         chmod 0755 $PACKETFENCE/addons/*.pl
         chmod 0755 $PACKETFENCE/addons/*.sh
         chmod 0755 $PACKETFENCE/addons/upgrade/*.pl
@@ -42,7 +41,8 @@ case "$1" in
         chmod ug+s $PACKETFENCE/bin/pfcmd
 
         chown pf:pf $PACKETFENCE
-        find $PACKETFENCE '(' -type d -or -type f ')' -not -name pfcmd -print0 | xargs -0 chown pf:pf
+        find $PACKETFENCE '(' -type d -or -type f ')' -not -name pfcmd -not -path "$PACKETFENCE/logs*" -print0 | xargs -0 chown pf:pf
+	chgrp pf $PACKETFENCE/logs
 
         # link to latest SQL schema
         if [ -h "$PACKETFENCE/db/pf-schema.sql" ]; then
@@ -52,15 +52,6 @@ case "$1" in
         VERSIONSQL=$(ls pf-schema-* |sort --version-sort -r | head -1)
         ln -f -s $VERSIONSQL ./pf-schema.sql
 
-        #Check if log files exist and create them with the correct owner
-        for fic_log in packetfence.log
-        do
-        if [ ! -e /usr/local/pf/logs/$fic_log ]; then
-            touch /usr/local/pf/logs/$fic_log
-            chown pf.pf /usr/local/pf/logs/$fic_log
-        fi
-        done
-
         #Make ssl certificate
         cd /usr/local/pf
         make conf/ssl/server.pem
@@ -140,8 +131,12 @@ case "$1" in
             echo "Setting packetfence.target as the default systemd target."
            /bin/systemctl set-default packetfence.target
         fi
+
+	systemctl daemon-reload
         echo "Restarting journald to enable persistent logging"
         /bin/systemctl restart systemd-journald
+	echo "Restarting rsyslog"
+        systemctl restart rsyslog
 
 	# get containers image and tag them locally
 	/usr/local/pf/containers/manage-images.sh
@@ -152,7 +147,6 @@ case "$1" in
         systemctl disable packetfence-iptables
         systemctl enable packetfence-haproxy-admin
         systemctl enable packetfence-tracking-config.path
-        systemctl daemon-reload
         systemctl restart docker
         systemctl start packetfence-config
         /usr/local/pf/bin/pfcmd generatemariadbconfig --force
@@ -162,9 +156,6 @@ case "$1" in
         printf '[client-server]\nsocket = /var/lib/mysql/mysql.sock\n' > /etc/mysql/mariadb.conf.d/999-socket-override.cnf
         sed -i 's#^socket\s*=.*#socket=/var/lib/mysql/mysql.sock#' /etc/mysql/my.cnf
 
-        echo "Restarting rsyslog"
-        systemctl restart rsyslog
-
         perl /usr/local/pf/addons/upgrade/add-default-params-to-auth.pl
         set +e
         /usr/local/pf/bin/pfcmd configreload

debian/rules

@@ -92,7 +92,7 @@ install: build
 	install -d -m0700 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/conf/ssl
 	install -d -m0700 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/conf/ssl/acme-challenge
 	install -d -m0700 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/raddb/sites-enabled
-	install -d -m2770 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/logs
+	install -d -m0750 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/logs
 	install -d -m2770 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/var/conf
 	install -d -m2770 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/var/cache
 	install -d -m2770 $(CURDIR)/debian/packetfence$(PREFIX)/$(NAME)/var/cache/ntlm_cache_users

docs/cluster/appendix.asciidoc

@@ -323,7 +323,7 @@ mysql -u root -p pf -h localhost
 ----
 
 If its not, make sure you check the MariaDB log
-([filename]`/usr/local/pf/logs/mariadb_error.log`)
+([filename]`/usr/local/pf/logs/mariadb.log`)
 
 ===== Sync nodes A and B
 
@@ -337,7 +337,7 @@ rm -fr /var/lib/mysql/*
 systemctl start packetfence-mariadb
 ----
 
-Should there be any issues during the sync, make sure you look into the MariaDB log ([filename]`/usr/local/pf/logs/mariadb_error.log`)
+Should there be any issues during the sync, make sure you look into the MariaDB log ([filename]`/usr/local/pf/logs/mariadb.log`)
 
 Once both nodes have completely synced (try connecting to it using the MariaDB
 command line), then you can break the cluster election command you have

docs/cluster/troubleshooting_a_cluster.asciidoc

@@ -33,7 +33,7 @@ Important variables:
   * `wsrep_last_committed`: Sequence number of the most recently committed transaction. You can identify the most advanced node with this value.
   * `wsrep_local_state_comment`: Current sync state of the cluster. A healthy state is 'Synced'. Refer to the Galera cluster documentation for the meaning of the other values this can have.
 
-In order for the cluster to be considered healthy, all nodes must be listed under `wsrep_incoming_addresses` and `wsrep_local_state_comment` must be `Synced`. Otherwise look in the MariaDB log ([filename]`/usr/local/pf/logs/mariadb_error.log`)
+In order for the cluster to be considered healthy, all nodes must be listed under `wsrep_incoming_addresses` and `wsrep_local_state_comment` must be `Synced`. Otherwise look in the MariaDB log ([filename]`/usr/local/pf/logs/mariadb.log`)
 
 === Automatic clustering resolution service: galera-autofix
 
@@ -123,7 +123,7 @@ systemctl start packetfence-mariadb
 You should then see `/var/lib/mysql` be populated again with the data and once
 MariaDB becomes available again on the server, it means the sync has
 completed. In case of issues, look in the MariaDB log file
-(`/usr/local/pf/logs/mariadb_error.log`)
+(`/usr/local/pf/logs/mariadb.log`)
 
 WARNING: After stopping the `packetfence-mariadb` service, be sure there is no more `mysql` process running.
 

docs/cluster/understanding_the_galera_cluster_synchronization.asciidoc

@@ -18,7 +18,7 @@ endif::[]
 
 The Galera cluster stack used by PacketFence resembles a lot to how a normal MariaDB Galera cluster behaves but it contains hooks to auto-correct some issues that can occur.
 
-NOTE: A lot of useful information is logged in the MariaDB log which can be found in `/usr/local/pf/logs/mariadb_error.log`
+NOTE: A lot of useful information is logged in the MariaDB log which can be found in `/usr/local/pf/logs/mariadb.log`
 
 === Quorum behavior
 

docs/installation/appendix.asciidoc

@@ -93,7 +93,7 @@ For Mariabackup:
   # chown -R mysql: /var/lib/mysql
   # service packetfence-mariadb start
 
-Should the service fail to start, make sure you look into the MariaDB error logs.
+Should the service fail to start, make sure you look into the MariaDB logs.
 
 [appendix]
 === How to restore a standalone PacketFence server ?

docs/installation/best_practices.asciidoc

@@ -24,7 +24,7 @@ IPTables is now entirely managed by PacketFence. However, if you need to perform
 
 === Log Rotations
 
-PacketFence can generate a lot of log entries in huge production environments. This is why we recommend to use `logrotate` to periodically rotate your logs. A working logrotate script is provided with the PacketFence package. This script is located under the `/usr/local/pf/packetfence.logrotate` file, and it's configured to do a daily log rotation and keeping old logs with compression. It has been added during PacketFence initial installation.
+PacketFence can generate a lot of log entries in huge production environments. This is why we recommend to use `logrotate` to periodically rotate your logs. A working logrotate script is provided with the PacketFence package. This script is located inside the logrotate directory (`/etc/logrotate.d/`), and it's configured to do a daily log rotation and keeping old logs with compression. It has been added during PacketFence initial installation.
 
 === Large Registration Network
 

docs/installation/troubleshooting_packetfence.asciidoc

@@ -24,20 +24,9 @@ PacketFence provides a RADIUS auditing module which allows you to be aware of al
 
 === Log files
 
-Here are the most important PacketFence log files:
-
-[options="compact"]
-* `/usr/local/pf/logs/packetfence.log` — PacketFence Core Log
-* `/usr/local/pf/logs/httpd.portal.access` — Apache – Captive Portal Access Log
-* `/usr/local/pf/logs/httpd.portal.error` — Apache – Captive Portal Error Log
-* `/usr/local/pf/logs/httpd.admin.access` — Apache – Web Admin/Services Access Log
-* `/usr/local/pf/logs/httpd.admin.error` — Apache – Web Admin/Services Error Log
-* `/usr/local/pf/logs/httpd.webservices.access` — Apache – Webservices Access Log
-* `/usr/local/pf/logs/httpd.webservices.error` — Apache – Webservices Error Log
-* `/usr/local/pf/logs/httpd.aaa.access` — Apache – AAA Access Log
-* `/usr/local/pf/logs/httpd.aaa.error` — Apache – AAA Error Log
-
-There are other log files in [filename]`/usr/local/pf/logs/` that could be relevant depending on what issue you are experiencing. Make sure you take a look at them.
+Log files are located under [filename]`/usr/local/pf/logs`. Except
+[filename]`packetfence.log` which contains logs from different services, each
+service has its own log file. You can see full list of log files available when using _Audit -> Live logs_ menu in web admin.
 
 The main logging configuration file is [filename]`/usr/local/pf/conf/log.conf`. It contains the configuration for the `packetfence.log` file (`Log::Log4Perl`) and you normally don't need to modify it. The logging configuration files for every service are located under [filename]`/usr/local/pf/conf/log.conf.d/`.
 

html/pfappserver/root/src/views/Status/dashboard/_config/logs.js

@@ -53,11 +53,11 @@ export default [
         ]
       },
       {
-        name: 'mariadb_error.log',
+        name: 'mariadb.log',
         items: [
           {
             title: 'Number of events', // i18n defer
-            metric: 'packetfence.logs.mariadb_error_log',
+            metric: 'packetfence.logs.mariadb_log',
             mode: modes.COMBINED,
             library: libraries.DYGRAPH,
             cols: 12
@@ -90,4 +90,4 @@ export default [
       }
     ]
   }
-]
+]

lib/pf/action.pm

@@ -90,7 +90,6 @@ use pf::class qw(class_view);
 use pf::security_event qw(security_event_force_close);
 use pf::Connection::ProfileFactory;
 use pf::constants::scan qw($POST_SCAN_SECURITY_EVENT_ID $PRE_SCAN_SECURITY_EVENT_ID $SCAN_SECURITY_EVENT_ID);
-use pf::file_paths qw($security_event_log);
 
 our $logger = get_logger();
 

lib/pf/cmd/pf/fixpermissions.pm

@@ -60,14 +60,13 @@ Fix the permissions on pf and fingerbank files
 sub action_all {
     my $pfcmd = "${bin_dir}/pfcmd";
     my @extra_var_dirs = map { catfile($var_dir,$_) } qw(run cache conf sessions redis_cache redis_queue);
-    _changeFilesToOwner('pf',@log_files, @stored_config_files, $install_dir, $bin_dir, $conf_dir, $var_dir, $lib_dir, $log_dir, $generated_conf_dir, $tt_compile_cache_dir, $pfconfig_cache_dir, @extra_var_dirs, $config_version_file, $iptable_config_file);
+    _changeFilesToOwner('pf', @stored_config_files, $install_dir, $bin_dir, $conf_dir, $var_dir, $lib_dir, $generated_conf_dir, $tt_compile_cache_dir, $pfconfig_cache_dir, @extra_var_dirs, $config_version_file, $iptable_config_file);
     _changePathToOwnerRecursive('pf', $html_dir);
     _changeFilesToOwner('root',$pfcmd);
     chmod($PFCMD_MODE, $pfcmd);
     chmod(0664, @stored_config_files, $iptable_config_file, $config_version_file);
-    chmod($DIR_MODE, $conf_dir, $var_dir, $log_dir, "$var_dir/redis_cache", "$var_dir/redis_queue");
+    chmod($DIR_MODE, $conf_dir, $var_dir, "$var_dir/redis_cache", "$var_dir/redis_queue");
     _fingerbank();
-    find({ wanted => \&wanted,untaint => 1}, $log_dir);
     print "Fixed permissions.\n";
     return $EXIT_SUCCESS;
 }
@@ -164,12 +163,6 @@ sub _fingerbank {
     fingerbank::Util::fix_permissions();
 }
 
-sub wanted {
-    return if $File::Find::name eq $log_dir;
-    my $perm = -d $File::Find::name ? 02775 : 0664;
-    chmod $perm, untaint_chain($File::Find::name);
-}
-
 =head1 AUTHOR
 
 Inverse inc. <info@inverse.ca>