From bc3967d9e5a33ff57628c78ffc9356a851ab9185 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 28 Nov 2023 14:23:20 +0100 Subject: [PATCH 1/2] Providing Backblaze support in Duplicity role. --- docs/_Sidebar.md | 1 + docs/roles/duplicity.md | 38 ++++++++++ roles/duplicity/README.md | 38 ++++++++++ roles/duplicity/defaults/main.yml | 5 +- roles/duplicity/tasks/main.yml | 6 +- .../templates/duplicity_backup-b2.j2 | 70 +++++++++++++++++++ ...icity_backup.j2 => duplicity_backup-s3.j2} | 18 ++--- .../duplicity/templates/duplicity_clean-b2.j2 | 39 +++++++++++ ...plicity_clean.j2 => duplicity_clean-s3.j2} | 10 +-- .../templates/duplicity_restore-b2.j2 | 44 ++++++++++++ ...ity_restore.j2 => duplicity_restore-s3.j2} | 10 +-- 11 files changed, 258 insertions(+), 21 deletions(-) create mode 100644 docs/roles/duplicity.md create mode 100644 roles/duplicity/README.md create mode 100644 roles/duplicity/templates/duplicity_backup-b2.j2 rename roles/duplicity/templates/{duplicity_backup.j2 => duplicity_backup-s3.j2} (73%) create mode 100644 roles/duplicity/templates/duplicity_clean-b2.j2 rename roles/duplicity/templates/{duplicity_clean.j2 => duplicity_clean-s3.j2} (68%) create mode 100644 roles/duplicity/templates/duplicity_restore-b2.j2 rename roles/duplicity/templates/{duplicity_restore.j2 => duplicity_restore-s3.j2} (72%) diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 7164f84be..81b07e8c1 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -39,6 +39,7 @@ - [Automated patching](/roles/ce_patcher) - [ce-provision](/roles/ce_provision) - [ClamAV](/roles/clamav) + - [Duplicity](/roles/duplicity) - [Firewall Config](/roles/firewall_config) - [Frontail](/roles/frontail) - [Gitlab](/roles/gitlab) diff --git a/docs/roles/duplicity.md b/docs/roles/duplicity.md new file mode 100644 index 000000000..7349f2eae --- /dev/null +++ b/docs/roles/duplicity.md @@ -0,0 +1,38 @@ +# Duplicity +Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux. + + + + + +## Default variables +```yaml +--- +duplicity: + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" + backend_url: "s3-eu-west-1.amazonaws.com" + bucketname: "somebucket" + dirs: + - name: "/boot" + rules: [] + - name: "/etc" + rules: [] + - name: "/opt" + rules: [] + - name: "/var" + rules: + - "+ /var/log/syslog*" + - "- /var" + exclude_other_filesystems: false + full_backup_frequency: "3M" + gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}" + install_dir: "/opt/duplicity" + mail_recipient: "foo@bar.com" + retention_period: "12M" + schedule: "0 0 * * *" + +``` + + diff --git a/roles/duplicity/README.md b/roles/duplicity/README.md new file mode 100644 index 000000000..7349f2eae --- /dev/null +++ b/roles/duplicity/README.md @@ -0,0 +1,38 @@ +# Duplicity +Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux. + + + + + +## Default variables +```yaml +--- +duplicity: + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" + backend_url: "s3-eu-west-1.amazonaws.com" + bucketname: "somebucket" + dirs: + - name: "/boot" + rules: [] + - name: "/etc" + rules: [] + - name: "/opt" + rules: [] + - name: "/var" + rules: + - "+ /var/log/syslog*" + - "- /var" + exclude_other_filesystems: false + full_backup_frequency: "3M" + gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}" + install_dir: "/opt/duplicity" + mail_recipient: "foo@bar.com" + retention_period: "12M" + schedule: "0 0 * * *" + +``` + + diff --git a/roles/duplicity/defaults/main.yml b/roles/duplicity/defaults/main.yml index 98fc178cc..87cb46ee6 100644 --- a/roles/duplicity/defaults/main.yml +++ b/roles/duplicity/defaults/main.yml @@ -1,7 +1,8 @@ --- duplicity: - aws_access_key_id: "somekey" - aws_secret_access_key: "somesecret" + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" backend_url: "s3-eu-west-1.amazonaws.com" bucketname: "somebucket" dirs: diff --git a/roles/duplicity/tasks/main.yml b/roles/duplicity/tasks/main.yml index 2fa297dde..b1d095b18 100644 --- a/roles/duplicity/tasks/main.yml +++ b/roles/duplicity/tasks/main.yml @@ -28,7 +28,7 @@ - name: Copy backup script in place. ansible.builtin.template: - src: duplicity_backup.j2 + src: "duplicity_backup-{{ duplicity.backend }}.j2" dest: "{{ duplicity.install_dir }}/bin/duplicity_backup" owner: root group: root @@ -36,7 +36,7 @@ - name: Copy restore script in place. ansible.builtin.template: - src: duplicity_restore.j2 + src: duplicity_restore-{{ duplicity.backend }}.j2 dest: "{{ duplicity.install_dir }}/bin/duplicity_restore" owner: root group: root @@ -44,7 +44,7 @@ - name: Copy clean-up script in place. ansible.builtin.template: - src: duplicity_clean.j2 + src: duplicity_clean-{{ duplicity.backend }}.j2 dest: "{{ duplicity.install_dir }}/bin/duplicity_clean" owner: root group: root diff --git a/roles/duplicity/templates/duplicity_backup-b2.j2 b/roles/duplicity/templates/duplicity_backup-b2.j2 new file mode 100644 index 000000000..0d502c0a3 --- /dev/null +++ b/roles/duplicity/templates/duplicity_backup-b2.j2 @@ -0,0 +1,70 @@ +#!/bin/bash + +# Duplicity Backup script + +# Declare and export secrets +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# How often should we make a full backup? Recommended: 3 months +FULL_BACKUPS="{{ duplicity.full_backup_frequency }}" + +# Remove old backups? 0 for no, 1 for yes +REMOVE_OLD_BACKUPS=1 + +# How often should we purge old backups? Recommended: 12 months. +REMOVE_OLDER_THAN="{{ duplicity.retention_period }}" + +# Args to pass to duplicity +{% if duplicity.exclude_other_filesystems %} +backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30" +{% else %} +backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30" +{% endif %} +maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force" + +# An array of directories to back up +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Backup code below. You should not need to edit anything here. + +# Loop over each dir and perform the backup. +for dir in ${DIRS[@]}; do + echo "Backing up $dir..." + + extra_options="" + if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then + extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" + fi + + # A special clause for /root. We don't want the local duplicity cache data + if [ $dir = "/root" ]; then + extra_options="$extra_options --exclude /root/.cache" + fi + + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + duplicity $backup_options $extra_options $dir $DEST || exit 1 + + if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then + # Do some maintenance on the remote end to clean up old backups + echo "Performing routine maintenance on $dir..." + duplicity $maintenance_options $DEST || exit 1 + fi +done + +unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/duplicity/templates/duplicity_backup.j2 b/roles/duplicity/templates/duplicity_backup-s3.j2 similarity index 73% rename from roles/duplicity/templates/duplicity_backup.j2 rename to roles/duplicity/templates/duplicity_backup-s3.j2 index 642ea656d..b8a8d8938 100644 --- a/roles/duplicity/templates/duplicity_backup.j2 +++ b/roles/duplicity/templates/duplicity_backup-s3.j2 @@ -3,9 +3,9 @@ # Duplicity Backup script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -15,13 +15,13 @@ fi ## Configurable variables # How often should we make a full backup? Recommended: 3 months -FULL_BACKUPS="{{duplicity.full_backup_frequency}}" +FULL_BACKUPS="{{ duplicity.full_backup_frequency }}" # Remove old backups? 0 for no, 1 for yes REMOVE_OLD_BACKUPS=1 # How often should we purge old backups? Recommended: 12 months. -REMOVE_OLDER_THAN="{{duplicity.retention_period}}" +REMOVE_OLDER_THAN="{{ duplicity.retention_period }}" # Args to pass to duplicity {% if duplicity.exclude_other_filesystems %} @@ -46,8 +46,8 @@ for dir in ${DIRS[@]}; do echo "Backing up $dir..." extra_options="" - if [ -f "{{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" ]; then - extra_options="--include-filelist {{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" + if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then + extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" fi # A special clause for /root. We don't want the local duplicity cache data @@ -55,7 +55,7 @@ for dir in ${DIRS[@]}; do extra_options="$extra_options --exclude /root/.cache" fi - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir duplicity $backup_options $extra_options $dir $DEST || exit 1 if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then @@ -66,3 +66,5 @@ for dir in ${DIRS[@]}; do done unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID diff --git a/roles/duplicity/templates/duplicity_clean-b2.j2 b/roles/duplicity/templates/duplicity_clean-b2.j2 new file mode 100644 index 000000000..99f982745 --- /dev/null +++ b/roles/duplicity/templates/duplicity_clean-b2.j2 @@ -0,0 +1,39 @@ +#!/bin/bash + +# Duplicity Cleanup script + +# Declare and export secrets +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# Args to pass to duplicity +cleanup_options="clean --force" + +# An array of directories to clean +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Cleanup code below. You should not need to edit anything here. + +# Loop over each dir and perform the clean. +for dir in ${DIRS[@]}; do + echo "Cleaning up $dir..." + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + duplicity $cleanup_options $DEST || exit 1 +done + +unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/duplicity/templates/duplicity_clean.j2 b/roles/duplicity/templates/duplicity_clean-s3.j2 similarity index 68% rename from roles/duplicity/templates/duplicity_clean.j2 rename to roles/duplicity/templates/duplicity_clean-s3.j2 index f61a84d36..f6298871e 100644 --- a/roles/duplicity/templates/duplicity_clean.j2 +++ b/roles/duplicity/templates/duplicity_clean-s3.j2 @@ -3,9 +3,9 @@ # Duplicity Cleanup script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -30,8 +30,10 @@ DIRS=( # Loop over each dir and perform the clean. for dir in ${DIRS[@]}; do echo "Cleaning up $dir..." - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir duplicity $cleanup_options $DEST || exit 1 done unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID diff --git a/roles/duplicity/templates/duplicity_restore-b2.j2 b/roles/duplicity/templates/duplicity_restore-b2.j2 new file mode 100644 index 000000000..74cf687f0 --- /dev/null +++ b/roles/duplicity/templates/duplicity_restore-b2.j2 @@ -0,0 +1,44 @@ +#!/bin/bash + +# Duplicity Restore script + +# Declare and export secrets +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# Directory to restore to +RESTORE_DIR=/tmp/restore-`date '+%F-%H%M'` + +# Restore to what point in time (ago) ? +#RESTORE_OPTIONS="-t 3D" + +# An array of directories to restore +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Restore code below. You should not need to edit anything here. + +mkdir -p $RESTORE_DIR + +# Loop over each dir and perform the restoration +for dir in ${DIRS[@]}; do + echo "Restoring $dir..." + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir +done + +unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/duplicity/templates/duplicity_restore.j2 b/roles/duplicity/templates/duplicity_restore-s3.j2 similarity index 72% rename from roles/duplicity/templates/duplicity_restore.j2 rename to roles/duplicity/templates/duplicity_restore-s3.j2 index aadac33ad..b3e568029 100644 --- a/roles/duplicity/templates/duplicity_restore.j2 +++ b/roles/duplicity/templates/duplicity_restore-s3.j2 @@ -3,9 +3,9 @@ # Duplicity Restore script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -36,8 +36,10 @@ mkdir -p $RESTORE_DIR # Loop over each dir and perform the restoration for dir in ${DIRS[@]}; do echo "Restoring $dir..." - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir done unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID From d72c333fd936a63007e5fcd149e11371ad3852e1 Mon Sep 17 00:00:00 2001 From: gregharvey Date: Tue, 28 Nov 2023 14:30:09 +0100 Subject: [PATCH 2/2] Moving Duplicity S3 options to an Ansible variable. --- docs/roles/duplicity.md | 1 + roles/duplicity/README.md | 1 + roles/duplicity/defaults/main.yml | 1 + roles/duplicity/templates/duplicity_backup-s3.j2 | 6 +++--- roles/duplicity/templates/duplicity_clean-s3.j2 | 2 +- roles/duplicity/templates/duplicity_restore-s3.j2 | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/roles/duplicity.md b/docs/roles/duplicity.md index 7349f2eae..43f4b6d7d 100644 --- a/docs/roles/duplicity.md +++ b/docs/roles/duplicity.md @@ -13,6 +13,7 @@ duplicity: access_key_id: "somekey" secret_access_key: "somesecret" backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: - name: "/boot" diff --git a/roles/duplicity/README.md b/roles/duplicity/README.md index 7349f2eae..43f4b6d7d 100644 --- a/roles/duplicity/README.md +++ b/roles/duplicity/README.md @@ -13,6 +13,7 @@ duplicity: access_key_id: "somekey" secret_access_key: "somesecret" backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: - name: "/boot" diff --git a/roles/duplicity/defaults/main.yml b/roles/duplicity/defaults/main.yml index 87cb46ee6..12cb44838 100644 --- a/roles/duplicity/defaults/main.yml +++ b/roles/duplicity/defaults/main.yml @@ -4,6 +4,7 @@ duplicity: access_key_id: "somekey" secret_access_key: "somesecret" backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: - name: "/boot" diff --git a/roles/duplicity/templates/duplicity_backup-s3.j2 b/roles/duplicity/templates/duplicity_backup-s3.j2 index b8a8d8938..8df13830b 100644 --- a/roles/duplicity/templates/duplicity_backup-s3.j2 +++ b/roles/duplicity/templates/duplicity_backup-s3.j2 @@ -25,11 +25,11 @@ REMOVE_OLDER_THAN="{{ duplicity.retention_period }}" # Args to pass to duplicity {% if duplicity.exclude_other_filesystems %} -backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 --s3-use-new-style --s3-european-buckets" +backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 {{ duplicity.s3_options }}" {% else %} -backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 --s3-use-new-style --s3-european-buckets" +backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 {{ duplicity.s3_options }}" {% endif %} -maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force --s3-use-new-style --s3-european-buckets" +maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force {{ duplicity.s3_options }}" # An array of directories to back up DIRS=( diff --git a/roles/duplicity/templates/duplicity_clean-s3.j2 b/roles/duplicity/templates/duplicity_clean-s3.j2 index f6298871e..82c6afe53 100644 --- a/roles/duplicity/templates/duplicity_clean-s3.j2 +++ b/roles/duplicity/templates/duplicity_clean-s3.j2 @@ -15,7 +15,7 @@ fi ## Configurable variables # Args to pass to duplicity -cleanup_options="clean --force --s3-use-new-style --s3-european-buckets" +cleanup_options="clean --force {{ duplicity.s3_options }}" # An array of directories to clean DIRS=( diff --git a/roles/duplicity/templates/duplicity_restore-s3.j2 b/roles/duplicity/templates/duplicity_restore-s3.j2 index b3e568029..fc4d505e7 100644 --- a/roles/duplicity/templates/duplicity_restore-s3.j2 +++ b/roles/duplicity/templates/duplicity_restore-s3.j2 @@ -19,7 +19,7 @@ RESTORE_DIR=/tmp/restore-`date '+%F-%H%M'` # Restore to what point in time (ago) ? #RESTORE_OPTIONS="-t 3D" -RESTORE_OPTIONS="--s3-use-new-style --s3-european-buckets" +RESTORE_OPTIONS="{{ duplicity.s3_options }}" # An array of directories to restore DIRS=(