Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Better logging #8

Merged
merged 5 commits into from
May 5, 2022
Merged

Better logging #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6354d46
Redirect log output file
deric May 5, 2022
b900204
[BC] rename logging variables, allow setting logging to console
deric May 5, 2022
41d93fa
Refactor composition of backup command, use epp template
deric May 5, 2022
90c8fe0
Refactor logging parameters
deric May 5, 2022
aa3f73d
Log rotation
deric May 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
25 changes: 25 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,31 @@ There are many shared parameters between `instance` and `catalog`. Such paramete
* `compress_level` `0-9` (defalt: `1`)
* `archive_timeout` Timeout in seconds for copying all remaining WAL files (default `300`).

### Logging

Parameters can be specified on `instance` level:

```yaml
pgprobackup::instance::log_dir: /var/log/pg_probackup
pgprobackup::instance::log_level_file: info
```

or for specific backup:
```yaml
pgprobackup::instance::backups:
common:
FULL:
log_level_file: verbose
```

* `log_dir` - by default stored to `backup dir/log`
* `log_file` - custom filename
* `redirect_console` redirect stdout & stderr to file (default: `false`)
* `log_console` - custom filename for console log (requires `redirect_console: true`)
* `log_level_file` - allowed values `off, error, warning, info, log, verbose`
* `log_level_console` - allowed values `off, error, warning, info, log, verbose`
* `log_rotation_size` - rotate logfile if its size exceeds this value; 0 disables; (default: 0) available units: 'kB', 'MB', 'GB', 'TB' (default: kB)
* `log_rotation_age` - rotate logfile if its size exceeds this value; 0 disables; (default: 0) available units: 'ms', 's', 'min', 'h', 'd' (default: min)

## Limitations

Expand Down
55 changes: 28 additions & 27 deletions manifests/catalog.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,28 @@
# @example
# include pgprobackup::catalog
class pgprobackup::catalog (
Stdlib::AbsolutePath $backup_dir = $pgprobackup::backup_dir,
Stdlib::AbsolutePath $log_dir = $pgprobackup::log_dir,
String $log_level = $pgprobackup::log_level,
String $logrotate_template = 'pgprobackup/logrotate.conf.erb',
String $exported_ipaddress = "${::ipaddress}/32",
String $user = $pgprobackup::backup_user,
String $group = $pgprobackup::backup_user,
String $dir_mode = '0750',
Enum['present', 'absent'] $user_ensure = 'present',
String $user_shell = '/bin/bash',
Boolean $manage_ssh_keys = $pgprobackup::manage_ssh_keys,
Boolean $manage_host_keys = $pgprobackup::manage_host_keys,
Boolean $manage_pgpass = $pgprobackup::manage_pgpass,
Boolean $manage_hba = $pgprobackup::manage_hba,
Boolean $manage_cron = $pgprobackup::manage_cron,
Boolean $purge_cron = true,
Optional[Integer] $uid = undef,
String $host_group = $pgprobackup::host_group,
Integer $hba_entry_order = 50,
String $ssh_key_fact = $::pgprobackup_catalog_key,
String $package_name = $pgprobackup::package_name,
Array[String] $versions = ['12'],
String $package_ensure = $pgprobackup::package_ensure,
Stdlib::AbsolutePath $backup_dir = $pgprobackup::backup_dir,
Optional[Stdlib::AbsolutePath] $log_dir = $pgprobackup::log_dir,
String $logrotate_template = 'pgprobackup/logrotate.conf.erb',
String $exported_ipaddress = "${::ipaddress}/32",
String $user = $pgprobackup::backup_user,
String $group = $pgprobackup::backup_user,
String $dir_mode = '0750',
Enum['present', 'absent'] $user_ensure = 'present',
String $user_shell = '/bin/bash',
Boolean $manage_ssh_keys = $pgprobackup::manage_ssh_keys,
Boolean $manage_host_keys = $pgprobackup::manage_host_keys,
Boolean $manage_pgpass = $pgprobackup::manage_pgpass,
Boolean $manage_hba = $pgprobackup::manage_hba,
Boolean $manage_cron = $pgprobackup::manage_cron,
Boolean $purge_cron = true,
Optional[Integer] $uid = undef,
String $host_group = $pgprobackup::host_group,
Integer $hba_entry_order = 50,
String $ssh_key_fact = $::pgprobackup_catalog_key,
String $package_name = $pgprobackup::package_name,
Array[String] $versions = ['12'],
String $package_ensure = $pgprobackup::package_ensure,
) inherits ::pgprobackup {

class {'pgprobackup::install':
Expand Down Expand Up @@ -88,10 +87,12 @@
require => File[$backup_dir],
}

file { $log_dir:
ensure => directory,
owner => $user,
group => $group,
if $log_dir {
file { $log_dir:
ensure => directory,
owner => $user,
group => $group,
}
}

if $manage_ssh_keys {
Expand Down
138 changes: 42 additions & 96 deletions manifests/cron_backup.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
String $remote_user,
Integer $remote_port,
Stdlib::AbsolutePath $backup_dir,
Stdlib::AbsolutePath $log_dir,
String $log_level,
Optional[Stdlib::AbsolutePath] $log_dir,
Optional[Pgprobackup::LogLevel] $log_level_file,
Optional[Pgprobackup::LogLevel] $log_level_console,
Optional[String] $log_file,
Optional[Integer] $retention_redundancy,
Optional[Integer] $retention_window,
Expand All @@ -35,103 +36,48 @@
Optional[Pgprobackup::Weekday] $weekday = '*',
Optional[Pgprobackup::Monthday] $monthday = undef,
Optional[String] $binary,
Boolean $redirect_console,
Optional[String] $log_console = undef,
Optional[String] $log_rotation_size = undef,
Optional[String] $log_rotation_age = undef,
){
if $binary {
$_binary = $binary
} else {
$_binary = "[ -x /usr/bin/pg_probackup-${version} ] && /usr/bin/pg_probackup-${version}"
}
$backup_cmd = "backup -B ${backup_dir}"

if $archive_wal {
$stream = ''
} else {
# with disabled WAL archiving, stream backup is needed
$stream = '--stream '
}

if $retention_redundancy {
$_retention_redundancy = " --retention-redundancy=${retention_redundancy}"
} else {
$_retention_redundancy = ''
}

if $retention_window {
$_retention_window = " --retention-window=${retention_window}"
} else {
$_retention_window = ''
}

if $retention_redundancy or $retention_window {
if $delete_expired {
$_dexpired = ' --delete-expired'
} else {
$_dexpired = ''
}
if $merge_expired {
$_mexpired = ' --merge-expired'
} else {
$_mexpired = ''
}
$expired = "${_dexpired}${_mexpired}"
} else {
$expired = ''
}

if $threads {
$_threads = " --threads=${threads}"
} else {
$_threads = ''
}

# replication slots
if $temp_slot {
$_temp_slot = ' --temp-slot'
} else {
$_temp_slot = ''
}

if $slot {
$_slot = " -S ${slot}"
} else {
$_slot = ''
}

if $validate {
$_validate = ''
} else {
$_validate = ' --no-validate'
}

$retention = "${_retention_redundancy}${_retention_window}${expired}"

if $compress_algorithm {
$_compress =" --compress-algorithm=${compress_algorithm} --compress-level=${compress_level}"
} else {
$_compress =''
}

if $archive_timeout {
$_timeout = " --archive-timeout=${archive_timeout}"
} else {
$_timeout = ''
}

if $log_file {
$_log_file = $log_file
} else {
# use file per db instance
$_log_file = "${id}.log"
}

$logging = "--log-filename=${_log_file} --log-level-file=${log_level} --log-directory=${log_dir}"

@@cron { "pgprobackup_${backup_type}_${server_address}-${host_group}":
command => @("CMD"/L),
${_binary} ${backup_cmd} --instance ${cluster} -b ${backup_type} ${stream}--remote-host=${server_address}\
--remote-user=${remote_user} --remote-port=${remote_port} -U ${db_user} -d ${db_name}\
${logging}${retention}${_threads}${_temp_slot}${_slot}${_validate}${_compress}${_timeout}
| -CMD
command => epp('pgprobackup/cron_backup.epp', {
id => $id,
cluster => $cluster,
db_name => $db_name,
db_user => $db_user,
version => $version,
host_group => $host_group,
backup_dir => $backup_dir,
backup_type => $backup_type,
backup_user => $backup_user,
server_address => $server_address,
delete_expired => $delete_expired,
retention_redundancy => $retention_redundancy,
retention_window => $retention_window,
merge_expired => $merge_expired,
threads => $threads,
temp_slot => $temp_slot,
slot => $slot,
validate => $validate,
compress_algorithm => $compress_algorithm,
compress_level => $compress_level,
archive_timeout => $archive_timeout,
archive_wal => $archive_wal,
log_dir => $log_dir,
log_file => $log_file,
log_level_file => $log_level_file,
log_level_console => $log_level_console,
remote_user => $remote_user,
remote_port => $remote_port,
binary => $binary,
redirect_console => $redirect_console,
log_console => $log_console,
log_rotation_size => $log_rotation_size,
log_rotation_age => $log_rotation_age,
}),
user => $backup_user,
weekday => $weekday,
hour => $hour,
Expand Down
33 changes: 16 additions & 17 deletions manifests/init.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,22 @@
# @example
# include pgprobackup
class pgprobackup(
String $package_name,
String $package_ensure = 'present',
String $host_group = 'common',
String $db_name = 'backup',
String $db_user = 'backup',
Boolean $debug_symbols = true,
Optional[String] $debug_suffix,
Stdlib::AbsolutePath $backup_dir = '/var/lib/pgbackup',
String $backup_user = 'pgbackup',
Stdlib::AbsolutePath $log_dir = '/var/lib/pgbackup/log',
String $log_level = 'info',
Boolean $manage_ssh_keys = true,
Boolean $manage_host_keys = true,
Boolean $manage_pgpass = true,
Boolean $manage_hba = true,
Boolean $manage_cron = true,
String $host_key_type = 'ecdsa-sha2-nistp256',
String $package_name,
String $package_ensure = 'present',
String $host_group = 'common',
String $db_name = 'backup',
String $db_user = 'backup',
Boolean $debug_symbols = true,
Optional[String] $debug_suffix,
Stdlib::AbsolutePath $backup_dir = '/var/lib/pgbackup',
String $backup_user = 'pgbackup',
Optional[Stdlib::AbsolutePath] $log_dir = undef,
Boolean $manage_ssh_keys = true,
Boolean $manage_host_keys = true,
Boolean $manage_pgpass = true,
Boolean $manage_hba = true,
Boolean $manage_cron = true,
String $host_key_type = 'ecdsa-sha2-nistp256',
) {

contain pgprobackup::repo
Expand Down
25 changes: 22 additions & 3 deletions manifests/instance.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,16 @@
# ssh port used for connection to the DB instance from catalog server
# @param binary
# custom script to be executed as backup command
# @param redirect_console
# Redirect console output to a log file (make sense especially with custom backup command)
# @param log_console
# File for storing console logs
# @param log_rotation_size
# rotate logfile if its size exceeds this value; 0 disables; (default: 0)
# available units: 'kB', 'MB', 'GB', 'TB' (default: kB)
# @param log_rotation_age
# rotate logfile if its age exceeds this value; 0 disables; (default: 0)
# available units: 'ms', 's', 'min', 'h', 'd' (default: min)
#
# @example
# include pgprobackup::instance
Expand All @@ -89,9 +99,10 @@
Stdlib::AbsolutePath $backup_dir = $pgprobackup::backup_dir,
String $backup_user = $pgprobackup::backup_user,
String $ssh_key_fact = $::pgprobackup_instance_key,
Stdlib::AbsolutePath $log_dir = $pgprobackup::log_dir,
Optional[Stdlib::AbsolutePath] $log_dir = $pgprobackup::log_dir,
Optional[String] $log_file = undef,
String $log_level = $pgprobackup::log_level,
Optional[Pgprobackup::LogLevel] $log_level_file = undef,
Optional[Pgprobackup::LogLevel] $log_level_console = undef,
Optional[Pgprobackup::Config] $backups = undef,
String $version = lookup('postgresql::globals::version'),
String $package_name = $pgprobackup::package_name,
Expand All @@ -108,6 +119,10 @@
Integer $compress_level = 1,
Optional[Integer] $archive_timeout = undef,
Optional[String] $binary = undef,
Boolean $redirect_console = false,
Optional[String] $log_console = undef,
Optional[String] $log_rotation_size = undef,
Optional[String] $log_rotation_age = undef,
) inherits pgprobackup {

$_cluster = $cluster ? {
Expand Down Expand Up @@ -277,10 +292,14 @@
archive_wal => $archive_wal,
log_dir => $log_dir,
log_file => $log_file,
log_level => $log_level,
log_level_file => $log_level_file,
log_level_console => $log_level_console,
remote_user => $remote_user,
remote_port => $remote_port,
binary => $binary,
redirect_console => $redirect_console,
log_rotation_size => $log_rotation_size,
log_rotation_age => $log_rotation_age,
})
}
} # manage_cron
Expand Down
22 changes: 15 additions & 7 deletions spec/classes/catalog_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,6 @@
mode: '0750')
}

it {
is_expected.to contain_file('/var/lib/pgbackup/log')
.with(ensure: 'directory',
owner: 'pgbackup',
group: 'pgbackup')
}

case os_facts[:os]['family']
when 'Debian'
it {
Expand Down Expand Up @@ -95,5 +88,20 @@
it { is_expected.to compile.and_raise_error(%r{Unsupported managed repository for osfamily}) }
end
end

context 'with log directory' do
let(:params) do
{
log_dir: '/var/log/pgbackup',
}
end

it {
is_expected.to contain_file('/var/log/pgbackup')
.with(ensure: 'directory',
owner: 'pgbackup',
group: 'pgbackup')
}
end
end
end