diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index 16c64086b..4a194080b 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -42,6 +42,7 @@ - [Automated patching](/roles/debian/ce_patcher) - [ce-provision](/roles/debian/ce_provision) - [ClamAV](/roles/debian/clamav) + - [Duplicity](/roles/debian/duplicity) - [Firewall Config](/roles/debian/firewall_config) - [Frontail](/roles/debian/frontail) - [Gitlab](/roles/debian/gitlab) diff --git a/docs/roles/debian/duplicity.md b/docs/roles/debian/duplicity.md new file mode 100644 index 000000000..9186201ed --- /dev/null +++ b/docs/roles/debian/duplicity.md @@ -0,0 +1,41 @@ +# Duplicity +Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux. + + + + + +## Default variables +```yaml +--- +duplicity: + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" + backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options + bucketname: "somebucket" + dirs: + - name: "/boot" + rules: [] + - name: "/etc" + rules: [] + - name: "/opt" + rules: [] + - name: "/var" + rules: + - "+ /var/log/syslog*" + - "- /var" + exclude_other_filesystems: false + full_backup_frequency: "3M" + gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}" + install_dir: "/opt/duplicity" + mail_recipient: "foo@bar.com" + retention_period: "12M" + # systemd timer settings + create_timer: true # sometimes you might want to trigger duplicity some other way than a systemd timer + on_calendar: "*-*-* 03:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + +``` + + diff --git a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml index d92e56a95..70597d699 100644 --- a/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml +++ b/roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml @@ -321,7 +321,6 @@ register: _aws_ec2_target_group_created when: - aws_ec2_autoscale_cluster.asg_refresh - - aws_ec2_autoscale_cluster.create_elb - name: Define default ALB listeners. ansible.builtin.set_fact: diff --git a/roles/aws/aws_vpc_route/tasks/route.yml b/roles/aws/aws_vpc_route/tasks/route.yml index 57e23f601..e353623af 100644 --- a/roles/aws/aws_vpc_route/tasks/route.yml +++ b/roles/aws/aws_vpc_route/tasks/route.yml @@ -7,6 +7,7 @@ gateway_id: "{{ route.gateway_id | default(omit) }}" nat_gateway_id: "{{ route.nat_gateway_id | default(omit) }}" network_interface_id: "{{ route.network_interface_id | default(omit) }}" + transit_gateway_id: "{{ route.transit_gateway_id | default(omit) }}" vpc_peering_connection_id: "{{ route.vpc_peering_connection_id | default(omit) }}" instance_id: "{{ route.instance_id | default(omit) }}" diff --git a/roles/debian/duplicity/README.md b/roles/debian/duplicity/README.md new file mode 100644 index 000000000..9186201ed --- /dev/null +++ b/roles/debian/duplicity/README.md @@ -0,0 +1,41 @@ +# Duplicity +Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux. + + + + + +## Default variables +```yaml +--- +duplicity: + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" + backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options + bucketname: "somebucket" + dirs: + - name: "/boot" + rules: [] + - name: "/etc" + rules: [] + - name: "/opt" + rules: [] + - name: "/var" + rules: + - "+ /var/log/syslog*" + - "- /var" + exclude_other_filesystems: false + full_backup_frequency: "3M" + gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}" + install_dir: "/opt/duplicity" + mail_recipient: "foo@bar.com" + retention_period: "12M" + # systemd timer settings + create_timer: true # sometimes you might want to trigger duplicity some other way than a systemd timer + on_calendar: "*-*-* 03:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events + +``` + + diff --git a/roles/debian/duplicity/defaults/main.yml b/roles/debian/duplicity/defaults/main.yml index 603b7611e..f85a94b36 100644 --- a/roles/debian/duplicity/defaults/main.yml +++ b/roles/debian/duplicity/defaults/main.yml @@ -1,8 +1,10 @@ --- duplicity: - aws_access_key_id: "somekey" - aws_secret_access_key: "somesecret" + backend: s3 # currently also support b2 for Backblaze + access_key_id: "somekey" + secret_access_key: "somesecret" backend_url: "s3-eu-west-1.amazonaws.com" + s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options bucketname: "somebucket" dirs: - name: "/boot" diff --git a/roles/debian/duplicity/tasks/main.yml b/roles/debian/duplicity/tasks/main.yml index dbae642bd..96581e0aa 100644 --- a/roles/debian/duplicity/tasks/main.yml +++ b/roles/debian/duplicity/tasks/main.yml @@ -28,7 +28,7 @@ - name: Copy backup script in place. ansible.builtin.template: - src: duplicity_backup.j2 + src: "duplicity_backup-{{ duplicity.backend }}.j2" dest: "{{ duplicity.install_dir }}/bin/duplicity_backup" owner: root group: root @@ -36,7 +36,7 @@ - name: Copy restore script in place. ansible.builtin.template: - src: duplicity_restore.j2 + src: duplicity_restore-{{ duplicity.backend }}.j2 dest: "{{ duplicity.install_dir }}/bin/duplicity_restore" owner: root group: root @@ -44,7 +44,7 @@ - name: Copy clean-up script in place. ansible.builtin.template: - src: duplicity_clean.j2 + src: duplicity_clean-{{ duplicity.backend }}.j2 dest: "{{ duplicity.install_dir }}/bin/duplicity_clean" owner: root group: root @@ -69,7 +69,7 @@ vars: timers: duplicity_backup: - timer_command: /usr/local/bin/duplicity_backup + timer_command: "{{ duplicity.install_dir }}/bin/duplicity_backup" timer_OnCalendar: "{{ duplicity.on_calendar }}" when: duplicity.create_timer diff --git a/roles/debian/duplicity/templates/duplicity_backup.j2 b/roles/debian/duplicity/templates/duplicity_backup-b2.j2 similarity index 62% rename from roles/debian/duplicity/templates/duplicity_backup.j2 rename to roles/debian/duplicity/templates/duplicity_backup-b2.j2 index 642ea656d..b41b992b8 100644 --- a/roles/debian/duplicity/templates/duplicity_backup.j2 +++ b/roles/debian/duplicity/templates/duplicity_backup-b2.j2 @@ -3,9 +3,9 @@ # Duplicity Backup script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -15,21 +15,21 @@ fi ## Configurable variables # How often should we make a full backup? Recommended: 3 months -FULL_BACKUPS="{{duplicity.full_backup_frequency}}" +FULL_BACKUPS="{{ duplicity.full_backup_frequency }}" # Remove old backups? 0 for no, 1 for yes REMOVE_OLD_BACKUPS=1 # How often should we purge old backups? Recommended: 12 months. -REMOVE_OLDER_THAN="{{duplicity.retention_period}}" +REMOVE_OLDER_THAN="{{ duplicity.retention_period }}" # Args to pass to duplicity {% if duplicity.exclude_other_filesystems %} -backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 --s3-use-new-style --s3-european-buckets" +backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30" {% else %} -backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 --s3-use-new-style --s3-european-buckets" +backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30" {% endif %} -maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force --s3-use-new-style --s3-european-buckets" +maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force" # An array of directories to back up DIRS=( @@ -46,8 +46,8 @@ for dir in ${DIRS[@]}; do echo "Backing up $dir..." extra_options="" - if [ -f "{{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" ]; then - extra_options="--include-filelist {{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" + if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then + extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" fi # A special clause for /root. We don't want the local duplicity cache data @@ -55,14 +55,16 @@ for dir in ${DIRS[@]}; do extra_options="$extra_options --exclude /root/.cache" fi - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir - duplicity $backup_options $extra_options $dir $DEST || exit 1 + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity $backup_options $extra_options $dir $DEST || exit 1 if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then # Do some maintenance on the remote end to clean up old backups echo "Performing routine maintenance on $dir..." - duplicity $maintenance_options $DEST || exit 1 + /usr/local/bin/duplicity $maintenance_options $DEST || exit 1 fi done unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/debian/duplicity/templates/duplicity_backup-s3.j2 b/roles/debian/duplicity/templates/duplicity_backup-s3.j2 new file mode 100644 index 000000000..993de8643 --- /dev/null +++ b/roles/debian/duplicity/templates/duplicity_backup-s3.j2 @@ -0,0 +1,70 @@ +#!/bin/bash + +# Duplicity Backup script + +# Declare and export secrets +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# How often should we make a full backup? Recommended: 3 months +FULL_BACKUPS="{{ duplicity.full_backup_frequency }}" + +# Remove old backups? 0 for no, 1 for yes +REMOVE_OLD_BACKUPS=1 + +# How often should we purge old backups? Recommended: 12 months. +REMOVE_OLDER_THAN="{{ duplicity.retention_period }}" + +# Args to pass to duplicity +{% if duplicity.exclude_other_filesystems %} +backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 {{ duplicity.s3_options }}" +{% else %} +backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 {{ duplicity.s3_options }}" +{% endif %} +maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force {{ duplicity.s3_options }}" + +# An array of directories to back up +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Backup code below. You should not need to edit anything here. + +# Loop over each dir and perform the backup. +for dir in ${DIRS[@]}; do + echo "Backing up $dir..." + + extra_options="" + if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then + extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" + fi + + # A special clause for /root. We don't want the local duplicity cache data + if [ $dir = "/root" ]; then + extra_options="$extra_options --exclude /root/.cache" + fi + + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity $backup_options $extra_options $dir $DEST || exit 1 + + if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then + # Do some maintenance on the remote end to clean up old backups + echo "Performing routine maintenance on $dir..." + /usr/local/bin/duplicity $maintenance_options $DEST || exit 1 + fi +done + +unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID diff --git a/roles/debian/duplicity/templates/duplicity_clean.j2 b/roles/debian/duplicity/templates/duplicity_clean-b2.j2 similarity index 59% rename from roles/debian/duplicity/templates/duplicity_clean.j2 rename to roles/debian/duplicity/templates/duplicity_clean-b2.j2 index f61a84d36..d8c8bb4b4 100644 --- a/roles/debian/duplicity/templates/duplicity_clean.j2 +++ b/roles/debian/duplicity/templates/duplicity_clean-b2.j2 @@ -3,9 +3,9 @@ # Duplicity Cleanup script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -15,7 +15,7 @@ fi ## Configurable variables # Args to pass to duplicity -cleanup_options="clean --force --s3-use-new-style --s3-european-buckets" +cleanup_options="clean --force" # An array of directories to clean DIRS=( @@ -30,8 +30,10 @@ DIRS=( # Loop over each dir and perform the clean. for dir in ${DIRS[@]}; do echo "Cleaning up $dir..." - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir - duplicity $cleanup_options $DEST || exit 1 + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity $cleanup_options $DEST || exit 1 done unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/debian/duplicity/templates/duplicity_clean-s3.j2 b/roles/debian/duplicity/templates/duplicity_clean-s3.j2 new file mode 100644 index 000000000..81b898fda --- /dev/null +++ b/roles/debian/duplicity/templates/duplicity_clean-s3.j2 @@ -0,0 +1,39 @@ +#!/bin/bash + +# Duplicity Cleanup script + +# Declare and export secrets +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# Args to pass to duplicity +cleanup_options="clean --force {{ duplicity.s3_options }}" + +# An array of directories to clean +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Cleanup code below. You should not need to edit anything here. + +# Loop over each dir and perform the clean. +for dir in ${DIRS[@]}; do + echo "Cleaning up $dir..." + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity $cleanup_options $DEST || exit 1 +done + +unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID diff --git a/roles/debian/duplicity/templates/duplicity_restore.j2 b/roles/debian/duplicity/templates/duplicity_restore-b2.j2 similarity index 64% rename from roles/debian/duplicity/templates/duplicity_restore.j2 rename to roles/debian/duplicity/templates/duplicity_restore-b2.j2 index aadac33ad..372431934 100644 --- a/roles/debian/duplicity/templates/duplicity_restore.j2 +++ b/roles/debian/duplicity/templates/duplicity_restore-b2.j2 @@ -3,9 +3,9 @@ # Duplicity Restore script # Declare and export secrets -export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}} -export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}} -export PASSPHRASE={{duplicity.gpg_passphrase}} +export PASSPHRASE={{ duplicity.gpg_passphrase }} +export B2_KEY_ID={{ duplicity.access_key_id }} +export B2_SECRET_KEY={{ duplicity.secret_access_key }} if [ ! `whoami` = "root" ] ; then echo "You must run this script as root" @@ -19,7 +19,6 @@ RESTORE_DIR=/tmp/restore-`date '+%F-%H%M'` # Restore to what point in time (ago) ? #RESTORE_OPTIONS="-t 3D" -RESTORE_OPTIONS="--s3-use-new-style --s3-european-buckets" # An array of directories to restore DIRS=( @@ -36,8 +35,10 @@ mkdir -p $RESTORE_DIR # Loop over each dir and perform the restoration for dir in ${DIRS[@]}; do echo "Restoring $dir..." - DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir - duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir + DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir done unset PASSPHRASE +unset B2_KEY_ID +unset B2_SECRET_KEY diff --git a/roles/debian/duplicity/templates/duplicity_restore-s3.j2 b/roles/debian/duplicity/templates/duplicity_restore-s3.j2 new file mode 100644 index 000000000..da038389f --- /dev/null +++ b/roles/debian/duplicity/templates/duplicity_restore-s3.j2 @@ -0,0 +1,45 @@ +#!/bin/bash + +# Duplicity Restore script + +# Declare and export secrets +export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }} +export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }} +export PASSPHRASE={{ duplicity.gpg_passphrase }} + +if [ ! `whoami` = "root" ] ; then + echo "You must run this script as root" + exit 1 +fi + +## Configurable variables + +# Directory to restore to +RESTORE_DIR=/tmp/restore-`date '+%F-%H%M'` + +# Restore to what point in time (ago) ? +#RESTORE_OPTIONS="-t 3D" +RESTORE_OPTIONS="{{ duplicity.s3_options }}" + +# An array of directories to restore +DIRS=( +{% for item in duplicity.dirs %} + {{ item.name }} +{% endfor %} +) + + +## Restore code below. You should not need to edit anything here. + +mkdir -p $RESTORE_DIR + +# Loop over each dir and perform the restoration +for dir in ${DIRS[@]}; do + echo "Restoring $dir..." + DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir + /usr/local/bin/duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir +done + +unset PASSPHRASE +unset AWS_SECRET_ACCESS_KEY +unset AWS_ACCESS_KEY_ID