Duplicity updates pr 1.x

docs/_Sidebar.md

@@ -39,6 +39,7 @@
      - [Automated patching](/roles/ce_patcher)
      - [ce-provision](/roles/ce_provision)
      - [ClamAV](/roles/clamav)
+     - [Duplicity](/roles/duplicity)
      - [Firewall Config](/roles/firewall_config)
      - [Frontail](/roles/frontail)
      - [Gitlab](/roles/gitlab)

docs/roles/duplicity.md

@@ -0,0 +1,39 @@
+# Duplicity
+Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+duplicity:
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
+  backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
+  bucketname: "somebucket"
+  dirs:
+    - name: "/boot"
+      rules: []
+    - name: "/etc"
+      rules: []
+    - name: "/opt"
+      rules: []
+    - name: "/var"
+      rules:
+        - "+ /var/log/syslog*"
+        - "- /var"
+  exclude_other_filesystems: false
+  full_backup_frequency: "3M"
+  gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}"
+  install_dir: "/opt/duplicity"
+  mail_recipient: "foo@bar.com"
+  retention_period: "12M"
+  schedule: "0 0 * * *"
+
+```
+
+<!--ENDROLEVARS-->

roles/duplicity/README.md

@@ -0,0 +1,39 @@
+# Duplicity
+Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+duplicity:
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
+  backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
+  bucketname: "somebucket"
+  dirs:
+    - name: "/boot"
+      rules: []
+    - name: "/etc"
+      rules: []
+    - name: "/opt"
+      rules: []
+    - name: "/var"
+      rules:
+        - "+ /var/log/syslog*"
+        - "- /var"
+  exclude_other_filesystems: false
+  full_backup_frequency: "3M"
+  gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}"
+  install_dir: "/opt/duplicity"
+  mail_recipient: "foo@bar.com"
+  retention_period: "12M"
+  schedule: "0 0 * * *"
+
+```
+
+<!--ENDROLEVARS-->

roles/duplicity/defaults/main.yml

@@ -1,8 +1,10 @@
 ---
 duplicity:
-  aws_access_key_id: "somekey"
-  aws_secret_access_key: "somesecret"
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
   backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
   bucketname: "somebucket"
   dirs:
     - name: "/boot"

roles/duplicity/tasks/main.yml

@@ -28,23 +28,23 @@
 
 - name: Copy backup script in place.
   ansible.builtin.template:
-    src: duplicity_backup.j2
+    src: "duplicity_backup-{{ duplicity.backend }}.j2"
     dest: "{{ duplicity.install_dir }}/bin/duplicity_backup"
     owner: root
     group: root
     mode: 0700
 
 - name: Copy restore script in place.
   ansible.builtin.template:
-    src: duplicity_restore.j2
+    src: duplicity_restore-{{ duplicity.backend }}.j2
     dest: "{{ duplicity.install_dir }}/bin/duplicity_restore"
     owner: root
     group: root
     mode: 0700
 
 - name: Copy clean-up script in place.
   ansible.builtin.template:
-    src: duplicity_clean.j2
+    src: duplicity_clean-{{ duplicity.backend }}.j2
     dest: "{{ duplicity.install_dir }}/bin/duplicity_clean"
     owner: root
     group: root

roles/duplicity/templates/duplicity_backup.j2 → roles/duplicity/templates/duplicity_backup-b2.j2

@@ -3,9 +3,9 @@
 # Duplicity Backup script
 
 # Declare and export secrets
-export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}}
-export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}}
-export PASSPHRASE={{duplicity.gpg_passphrase}}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+export B2_KEY_ID={{ duplicity.access_key_id }}
+export B2_SECRET_KEY={{ duplicity.secret_access_key }}
 
 if [ ! `whoami` = "root" ] ; then
   echo "You must run this script as root"
@@ -15,21 +15,21 @@ fi
 ## Configurable variables
 
 # How often should we make a full backup? Recommended: 3 months
-FULL_BACKUPS="{{duplicity.full_backup_frequency}}"
+FULL_BACKUPS="{{ duplicity.full_backup_frequency }}"
 
 # Remove old backups? 0 for no, 1 for yes
 REMOVE_OLD_BACKUPS=1
 
 # How often should we purge old backups? Recommended: 12 months.
-REMOVE_OLDER_THAN="{{duplicity.retention_period}}"
+REMOVE_OLDER_THAN="{{ duplicity.retention_period }}"
 
 # Args to pass to duplicity
 {% if duplicity.exclude_other_filesystems %}
-backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 --s3-use-new-style --s3-european-buckets"
+backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30"
 {% else %}
-backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 --s3-use-new-style --s3-european-buckets"
+backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30"
 {% endif %}
-maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force --s3-use-new-style --s3-european-buckets"
+maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force"
 
 # An array of directories to back up
 DIRS=(
@@ -46,16 +46,16 @@ for dir in ${DIRS[@]}; do
   echo "Backing up $dir..."
 
   extra_options=""
-  if [ -f "{{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" ]; then
-    extra_options="--include-filelist {{duplicity.install_dir}}/etc/$dir-include-exclude-filelist"
+  if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then
+    extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist"
   fi
 
   # A special clause for /root. We don't want the local duplicity cache data
   if [ $dir = "/root" ]; then
     extra_options="$extra_options --exclude /root/.cache"
   fi
 
-  DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir
+  DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir
   duplicity $backup_options $extra_options $dir $DEST || exit 1
 
   if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then
@@ -66,3 +66,5 @@ for dir in ${DIRS[@]}; do
 done
 
 unset PASSPHRASE
+unset B2_KEY_ID
+unset B2_SECRET_KEY

roles/duplicity/templates/duplicity_backup-s3.j2

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Duplicity Backup script
+
+# Declare and export secrets
+export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }}
+export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+
+if [ ! `whoami` = "root" ] ; then
+  echo "You must run this script as root"
+  exit 1
+fi
+
+## Configurable variables
+
+# How often should we make a full backup? Recommended: 3 months
+FULL_BACKUPS="{{ duplicity.full_backup_frequency }}"
+
+# Remove old backups? 0 for no, 1 for yes
+REMOVE_OLD_BACKUPS=1
+
+# How often should we purge old backups? Recommended: 12 months.
+REMOVE_OLDER_THAN="{{ duplicity.retention_period }}"
+
+# Args to pass to duplicity
+{% if duplicity.exclude_other_filesystems %}
+backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 {{ duplicity.s3_options }}"
+{% else %}
+backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 {{ duplicity.s3_options }}"
+{% endif %}
+maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force {{ duplicity.s3_options }}"
+
+# An array of directories to back up
+DIRS=(
+{% for item in duplicity.dirs %}
+  {{ item.name }}
+{% endfor %}
+)
+
+
+## Backup code below. You should not need to edit anything here.
+
+# Loop over each dir and perform the backup.
+for dir in ${DIRS[@]}; do
+  echo "Backing up $dir..."
+
+  extra_options=""
+  if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then
+    extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist"
+  fi
+
+  # A special clause for /root. We don't want the local duplicity cache data
+  if [ $dir = "/root" ]; then
+    extra_options="$extra_options --exclude /root/.cache"
+  fi
+
+  DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir
+  duplicity $backup_options $extra_options $dir $DEST || exit 1
+
+  if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then
+    # Do some maintenance on the remote end to clean up old backups
+    echo "Performing routine maintenance on $dir..."
+    duplicity $maintenance_options $DEST || exit 1
+  fi
+done
+
+unset PASSPHRASE
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_ACCESS_KEY_ID

roles/duplicity/templates/duplicity_clean.j2 → roles/duplicity/templates/duplicity_clean-b2.j2

@@ -3,9 +3,9 @@
 # Duplicity Cleanup script
 
 # Declare and export secrets
-export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}}
-export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}}
-export PASSPHRASE={{duplicity.gpg_passphrase}}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+export B2_KEY_ID={{ duplicity.access_key_id }}
+export B2_SECRET_KEY={{ duplicity.secret_access_key }}
 
 if [ ! `whoami` = "root" ] ; then
   echo "You must run this script as root"
@@ -15,7 +15,7 @@ fi
 ## Configurable variables
 
 # Args to pass to duplicity
-cleanup_options="clean --force --s3-use-new-style --s3-european-buckets"
+cleanup_options="clean --force"
 
 # An array of directories to clean
 DIRS=(
@@ -30,8 +30,10 @@ DIRS=(
 # Loop over each dir and perform the clean.
 for dir in ${DIRS[@]}; do
   echo "Cleaning up $dir..."
-  DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir
+  DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir
   duplicity $cleanup_options $DEST || exit 1
 done
 
 unset PASSPHRASE
+unset B2_KEY_ID
+unset B2_SECRET_KEY

roles/duplicity/templates/duplicity_clean-s3.j2

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Duplicity Cleanup script
+
+# Declare and export secrets
+export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }}
+export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+
+if [ ! `whoami` = "root" ] ; then
+  echo "You must run this script as root"
+  exit 1
+fi
+
+## Configurable variables
+
+# Args to pass to duplicity
+cleanup_options="clean --force {{ duplicity.s3_options }}"
+
+# An array of directories to clean
+DIRS=(
+{% for item in duplicity.dirs %}
+  {{ item.name }}
+{% endfor %}
+)
+
+
+## Cleanup code below. You should not need to edit anything here.
+
+# Loop over each dir and perform the clean.
+for dir in ${DIRS[@]}; do
+  echo "Cleaning up $dir..."
+  DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir
+  duplicity $cleanup_options $DEST || exit 1
+done
+
+unset PASSPHRASE
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_ACCESS_KEY_ID

roles/duplicity/templates/duplicity_restore.j2 → roles/duplicity/templates/duplicity_restore-b2.j2

@@ -3,9 +3,9 @@
 # Duplicity Restore script
 
 # Declare and export secrets
-export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}}
-export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}}
-export PASSPHRASE={{duplicity.gpg_passphrase}}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+export B2_KEY_ID={{ duplicity.access_key_id }}
+export B2_SECRET_KEY={{ duplicity.secret_access_key }}
 
 if [ ! `whoami` = "root" ] ; then
   echo "You must run this script as root"
@@ -19,7 +19,6 @@ RESTORE_DIR=/tmp/restore-`date '+%F-%H%M'`
 
 # Restore to what point in time (ago) ?
 #RESTORE_OPTIONS="-t 3D"
-RESTORE_OPTIONS="--s3-use-new-style --s3-european-buckets"
 
 # An array of directories to restore
 DIRS=(
@@ -36,8 +35,10 @@ mkdir -p $RESTORE_DIR
 # Loop over each dir and perform the restoration
 for dir in ${DIRS[@]}; do
   echo "Restoring $dir..."
-  DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir
+  DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir
   duplicity restore $RESTORE_OPTIONS $DEST $RESTORE_DIR$dir
 done
 
 unset PASSPHRASE
+unset B2_KEY_ID
+unset B2_SECRET_KEY