From f3858c1012f828aff5f2bc52abd2e285006bb2a9 Mon Sep 17 00:00:00 2001 From: hashworks Date: Sat, 2 Apr 2022 18:37:00 +0200 Subject: [PATCH 1/3] Move configuration file to script template Since the backup script is a template anyway it is unecessary to split move some configuration to additional files. --- manifests/config.pp | 7 ++----- spec/classes/init_spec.rb | 3 +-- templates/backup-sh-conf.sh.epp | 18 ------------------ templates/borg-backup.sh.epp | 29 ++++++++++++----------------- 4 files changed, 15 insertions(+), 42 deletions(-) delete mode 100644 templates/backup-sh-conf.sh.epp diff --git a/manifests/config.pp b/manifests/config.pp index a8820dd..43b377b 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -37,12 +37,9 @@ ensure => 'file', content => epp("${module_name}/borg-restore.cfg.epp", { 'backupdestdir' => $backupdestdir, }), } - # config file with all excludes and includes + # config file is deprecated and should be absent file { '/etc/backup-sh-conf.sh': - ensure => 'file', - content => epp("${module_name}/backup-sh-conf.sh.epp"), - owner => 'root', - group => 'root', + ensure => 'absent', } # create the backup key for a user ssh_keygen { 'root_borg': diff --git a/spec/classes/init_spec.rb b/spec/classes/init_spec.rb index 86fd2ba..8cdc21a 100644 --- a/spec/classes/init_spec.rb +++ b/spec/classes/init_spec.rb @@ -22,7 +22,6 @@ context 'with all defaults' do it { is_expected.to compile.with_all_deps } - it { is_expected.to contain_file('/etc/backup-sh-conf.sh') } it { is_expected.to contain_file('/etc/borg') } it { is_expected.to contain_file('/etc/profile.d/borg.sh') } it { is_expected.to contain_file('/usr/local/bin/borg-backup') } @@ -217,7 +216,7 @@ } end - it { is_expected.to contain_file('/etc/backup-sh-conf.sh').with_content(%r{^"/path with/spaces"$}) } + it { is_expected.to contain_file('/usr/local/bin/borg-backup').with_content(%r{^"/path with/spaces"$}) } end context 'without exclude_pattern' do diff --git a/templates/backup-sh-conf.sh.epp b/templates/backup-sh-conf.sh.epp deleted file mode 100644 index b129bb0..0000000 --- a/templates/backup-sh-conf.sh.epp +++ /dev/null @@ -1,18 +0,0 @@ -# THIS FILE IS MANAGED BY PUPPET - -excludeMountpoints=( -<% $borg::excludes.each |$key| { -%> -"<%= $key %>" -<% } -%> -<% $borg::additional_excludes.each |$key| { -%> -"<%= $key %>" -<% } -%> -) -includeMountpoints=( -<% $borg::includes.each |$key| { -%> -"<%= $key %>" -<% } -%> -<% $borg::additional_includes.each |$key| { -%> -"<%= $key %>" -<% } -%> -) diff --git a/templates/borg-backup.sh.epp b/templates/borg-backup.sh.epp index e71af82..5b85f89 100644 --- a/templates/borg-backup.sh.epp +++ b/templates/borg-backup.sh.epp @@ -17,8 +17,6 @@ # # Important steps: # - define a host "backup" in root's .ssh/config -# - You can override variables from main() as well as the pre_backup() and post_backup() functions in -# $XDG_CONFIG_HOME/backup-sh-conf.sh or /etc/backup-sh-conf.sh # - As root run `BORG_PASSPHRASE='' borg init -v --encryption=keyfile backup:<%= $backupdestdir %>` # (note that zsh uses $HOST instead of $HOSTNAME) # - If you want, increase the max_segment_size in @@ -67,20 +65,22 @@ main() { # These mountpoints will be excluded. Mountpoints not listed in either this # or the includeMountpoints variable below will throw an error excludeMountpoints=( - /tmp - /sys - /dev - /proc - /run - /media + <% $borg::excludes.each |$key| { -%> + "<%= $key %>" + <% } -%> + <% $borg::additional_excludes.each |$key| { -%> + "<%= $key %>" + <% } -%> ) # These mountpoints will be included includeMountpoints=( - / - /boot - /home - /mnt/data + <% $borg::includes.each |$key| { -%> + "<%= $key %>" + <% } -%> + <% $borg::additional_includes.each |$key| { -%> + "<%= $key %>" + <% } -%> ) # List of patterns that should be excluded. This supports shell globbing as @@ -91,11 +91,6 @@ main() { <% } -%> EOF - for configfile in "${XDG_CONFIG_HOME:-$HOME/.config}/backup-sh-conf.sh" /etc/backup-sh-conf.sh; do - if [[ -e "$configfile" ]]; then - source "$configfile" - fi - done exclude_mountpoints echo "$excludeList" > "$TMPDIR/exclude-list-borg" From 2b934708cd54906781f722ebed902a04f4b4f903 Mon Sep 17 00:00:00 2001 From: hashworks Date: Sat, 2 Apr 2022 18:39:28 +0200 Subject: [PATCH 2/3] Add small updates from upstream `--numeric-owner` is deprecated and replaced by `--numeric-ids`. See https://git.server-speed.net/users/flo/bin/tree/backup.sh --- templates/borg-backup.sh.epp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/borg-backup.sh.epp b/templates/borg-backup.sh.epp index 5b85f89..ae4fc90 100644 --- a/templates/borg-backup.sh.epp +++ b/templates/borg-backup.sh.epp @@ -147,13 +147,13 @@ backup_borg() { local dst=$2 local -a options=( --verbose - --numeric-owner + --numeric-ids --compression "<%= $compression %>" --exclude-from "$TMPDIR/exclude-list-borg" ) if tty -s; then - options+=(--progress) + options+=(--progress --list --filter AME) fi borg create "${options[@]}" --stats --verbose "$dst::backup-$(date "+%Y%m%d-%H%M%S")" "$src" From 51ed9db8badd993c9be26538b58a4119fd49ea2b Mon Sep 17 00:00:00 2001 From: hashworks Date: Sat, 2 Apr 2022 18:42:00 +0200 Subject: [PATCH 3/3] Add support for snapshots using pre/post scripts Snapshots should be created using `pre_backup_script` and cleaned up with `post_backup_script`. The `source_paths` should be adjusted to match the snapshot mount point. --- REFERENCE.md | 36 ++++++++++++++++++++++++++++++++++++ manifests/config.pp | 20 +++++++++++--------- manifests/init.pp | 16 ++++++++++++++++ spec/classes/init_spec.rb | 4 ++-- templates/borg-backup.sh.epp | 31 ++++++++++++++++++++++++------- 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 2f0382f..504fe81 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -40,12 +40,14 @@ The following parameters are available in the `borg` class: * [`keep_daily`](#keep_daily) * [`keep_within`](#keep_within) * [`compression`](#compression) +* [`source_paths`](#source_paths) * [`excludes`](#excludes) * [`includes`](#includes) * [`backupserver`](#backupserver) * [`install_restore_script`](#install_restore_script) * [`restore_script_path`](#restore_script_path) * [`backupdestdir`](#backupdestdir) +* [`backupdatadir`](#backupdatadir) * [`absolutebackupdestdir`](#absolutebackupdestdir) * [`manage_repository`](#manage_repository) * [`exclude_pattern`](#exclude_pattern) @@ -67,6 +69,8 @@ The following parameters are available in the `borg` class: * [`wants`](#wants) * [`requires`](#requires) * [`after`](#after) +* [`pre_backup_script`](#pre_backup_script) +* [`post_backup_script`](#post_backup_script) ##### `package_name` @@ -154,6 +158,14 @@ Compression method and level to use. See the output of `borg help compression` f Default value: `'lz4'` +##### `source_paths` + +Data type: `Array[String[1]]` + +A list of relative or absolute paths to backup. + +Default value: `['/']` + ##### `excludes` Data type: `Array[Stdlib::Absolutepath]` @@ -196,6 +208,14 @@ The path on the remote server where the backups should be written to. $username Default value: `'borg'` +##### `backupdatadir` + +Data type: `Stdlib::Absolutepath` + +The path where additional backup data should be stored. + +Default value: `'/root/backup-data/'` + ##### `absolutebackupdestdir` Data type: `Optional[String[1]]` @@ -360,3 +380,19 @@ Array of units that should be started before the borg-backup service Default value: `['network-online.target']` +##### `pre_backup_script` + +Data type: `Optional[String[1]]` + +BASH code to be executed before the backup job starts. If you wish to use snapshots, create them here. + +Default value: ``undef`` + +##### `post_backup_script` + +Data type: `Optional[String[1]]` + +BASH code to be executed after the backup job has finished. If you need to perform any cleanup do so here. + +Default value: ``undef`` + diff --git a/manifests/config.pp b/manifests/config.pp index 43b377b..02900e7 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -12,15 +12,17 @@ ensure => 'file', content => epp("${module_name}/borg-backup.sh.epp", { - 'manage_prune' => $borg::manage_prune, - 'keep_within' => $borg::keep_within, - 'keep_daily' => $borg::keep_daily, - 'keep_weekly' => $borg::keep_weekly, - 'keep_monthly' => $borg::keep_monthly, - 'keep_yearly' => $borg::keep_yearly, - 'compression' => $borg::compression, - 'backupdestdir' => $backupdestdir, - 'exclude_pattern' => $borg::exclude_pattern + $borg::additional_exclude_pattern, + 'manage_prune' => $borg::manage_prune, + 'keep_within' => $borg::keep_within, + 'keep_daily' => $borg::keep_daily, + 'keep_weekly' => $borg::keep_weekly, + 'keep_monthly' => $borg::keep_monthly, + 'keep_yearly' => $borg::keep_yearly, + 'compression' => $borg::compression, + 'backupdestdir' => $backupdestdir, + 'backupdatadir' => $borg::backupdatadir, + 'pre_backup_script' => $borg::pre_backup_script, + 'post_backup_script' => $borg::post_backup_script, }), mode => '0755', owner => 'root', diff --git a/manifests/init.pp b/manifests/init.pp index 7f44d38..b12e842 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -33,6 +33,9 @@ # @param compression # Compression method and level to use. See the output of `borg help compression` for available options. # +# @param source_paths +# A list of relative or absolute paths to backup. +# # @param excludes # list of default mountpoints that should be excluded from backups. Every mountpoint needs to be explicitly excluded or included. See also the additional_excludes parameter. # @@ -51,6 +54,9 @@ # @param backupdestdir # The path on the remote server where the backups should be written to. $username will be prepended # +# @param backupdatadir +# The path where additional backup data should be stored. +# # @param absolutebackupdestdir # By defaults, backups will be written on the remote host to $username/$backupdestdir. if $absolutebackupdestdir is set this will be used instead # @@ -114,6 +120,12 @@ # @param after # Array of units that should be started before the borg-backup service # +# @param pre_backup_script +# BASH code to be executed before the backup job starts. If you wish to use snapshots, create them here. +# +# @param post_backup_script +# BASH code to be executed after the backup job has finished. If you need to perform any cleanup do so here. +# # @see https://metacpan.org/pod/App::BorgRestore # class borg ( @@ -133,9 +145,11 @@ Integer[0] $keep_daily = 60, Integer[0] $keep_within = 30, String[1] $compression = 'lz4', + Array[String[1]] $source_paths = ['/'], Array[Stdlib::Absolutepath] $excludes = ['/tmp', '/sys', '/dev', '/proc', '/run', '/media', '/var/lib/nfs/rpc_pipefs'], Array[Stdlib::Absolutepath] $includes = ['/', '/boot', '/boot/efi', '/boot/EFI', '/var/log'], String[1] $backupdestdir = 'borg', + Stdlib::Absolutepath $backupdatadir = '/root/backup-data/', Optional[String[1]] $absolutebackupdestdir = undef, Array[String[1]] $exclude_pattern = ['sh:/home/*/.cache/*', 'sh:/root/.cache/*', 'sh:/var/cache/pacman/pkg/*'], Array[String[1]] $additional_exclude_pattern = [], @@ -155,6 +169,8 @@ Array[String[1]] $wants = ['network-online.target'], Array[String[1]] $requires = [], Array[String[1]] $after = ['network-online.target'], + Optional[String[1]] $pre_backup_script = undef, + Optional[String[1]] $post_backup_script = undef, ) { contain borg::install contain borg::config diff --git a/spec/classes/init_spec.rb b/spec/classes/init_spec.rb index 8cdc21a..684322f 100644 --- a/spec/classes/init_spec.rb +++ b/spec/classes/init_spec.rb @@ -205,7 +205,7 @@ it { is_expected.to contain_file('/etc/borg').with_content(%r{^REPOSITORY=backup:/some/other/path$}) } it { is_expected.to contain_file('/etc/profile.d/borg.sh').with_content(%r{^export BORG_REPO=backup:/some/other/path$}) } - it { is_expected.to contain_file('/usr/local/bin/borg-backup').with_content(%r{\s*borg_repo="backup:/some/other/path"$}) } + it { is_expected.to contain_file('/usr/local/bin/borg-backup').with_content(%r{^\s*borg_repo="backup:/some/other/path"$}) } end context 'with additional excludes' do @@ -216,7 +216,7 @@ } end - it { is_expected.to contain_file('/usr/local/bin/borg-backup').with_content(%r{^"/path with/spaces"$}) } + it { is_expected.to contain_file('/usr/local/bin/borg-backup').with_content(%r{^\s*"/path with/spaces"$}) } end context 'without exclude_pattern' do diff --git a/templates/borg-backup.sh.epp b/templates/borg-backup.sh.epp index ae4fc90..c622eeb 100644 --- a/templates/borg-backup.sh.epp +++ b/templates/borg-backup.sh.epp @@ -6,7 +6,9 @@ Integer[0] $keep_within, String[1] $compression, String[1] $backupdestdir, - Array[String[1]] $exclude_pattern, + Stdlib::Absolutepath $backupdatadir, + Optional[String[1]] $pre_backup_script, + Optional[String[1]] $post_backup_script, | -%> #!/bin/bash # @@ -46,7 +48,7 @@ # modified by Tim 'bastelfreak' Meusel ## -set -e +set -eu # set PATH by hand so we ensure that the puppetlabs stuff is present PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/opt/puppetlabs/bin:/bin:/sbin" @@ -89,21 +91,29 @@ main() { <% $borg::exclude_pattern.each |$key| { -%> <%= $key %> <% } -%> +<% $borg::additional_exclude_pattern.each |$key| { -%> +<%= $key %> +<% } -%> EOF + source_paths=( + <% $borg::source_paths.each |$key| { -%> + "<%= $key %>" + <% } -%> + ) exclude_mountpoints echo "$excludeList" > "$TMPDIR/exclude-list-borg" run_if_exists pre_backup - backup_borg / "$borg_repo" + backup_borg "$borg_repo" "${source_paths[@]}" run_if_exists post_backup } # This is called before creating the backup pre_backup() { # save some data that's useful for restores - local backupDataDir=/root/backup-data/ + local backupDataDir="<%= $backupdatadir %>" mkdir -p "$backupDataDir" if [ $(command -v fdisk) ]; then fdisk -l > "$backupDataDir/fdisk" @@ -131,6 +141,9 @@ pre_backup() { fi # If you wish to use snapshots, create them here +<% if $pre_backup_script { -%> +<%= $pre_backup_script %> +<% } -%> return } @@ -139,12 +152,16 @@ pre_backup() { post_backup() { # If you need to perform any cleanup do so here +<% if $post_backup_script { -%> +<%= $post_backup_script %> +<% } -%> + return } backup_borg() { - local src=$1 - local dst=$2 + local dst=$1; shift + local src=("$@") local -a options=( --verbose --numeric-ids @@ -156,7 +173,7 @@ backup_borg() { options+=(--progress --list --filter AME) fi - borg create "${options[@]}" --stats --verbose "$dst::backup-$(date "+%Y%m%d-%H%M%S")" "$src" + borg create "${options[@]}" --stats --verbose "$dst::backup-$(date "+%Y%m%d-%H%M%S")" "${src[@]}" <% if $manage_prune { -%> # keep all backups from the last <%= $keep_within %> days # keep at least one backup for each day for <%= $keep_daily %> days