Duplicity updates pr 2.x

docs/_Sidebar.md

@@ -42,6 +42,7 @@
        - [Automated patching](/roles/debian/ce_patcher)
        - [ce-provision](/roles/debian/ce_provision)
        - [ClamAV](/roles/debian/clamav)
+       - [Duplicity](/roles/debian/duplicity)
        - [Firewall Config](/roles/debian/firewall_config)
        - [Frontail](/roles/debian/frontail)
        - [Gitlab](/roles/debian/gitlab)

docs/roles/debian/duplicity.md

@@ -0,0 +1,41 @@
+# Duplicity
+Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+duplicity:
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
+  backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
+  bucketname: "somebucket"
+  dirs:
+    - name: "/boot"
+      rules: []
+    - name: "/etc"
+      rules: []
+    - name: "/opt"
+      rules: []
+    - name: "/var"
+      rules:
+        - "+ /var/log/syslog*"
+        - "- /var"
+  exclude_other_filesystems: false
+  full_backup_frequency: "3M"
+  gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}"
+  install_dir: "/opt/duplicity"
+  mail_recipient: "foo@bar.com"
+  retention_period: "12M"
+  # systemd timer settings
+  create_timer: true # sometimes you might want to trigger duplicity some other way than a systemd timer
+  on_calendar: "*-*-* 03:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events
+
+```
+
+<!--ENDROLEVARS-->

roles/aws/aws_ec2_autoscale_cluster/tasks/main.yml

@@ -321,7 +321,6 @@
   register: _aws_ec2_target_group_created
   when:
     - aws_ec2_autoscale_cluster.asg_refresh
-    - aws_ec2_autoscale_cluster.create_elb
 
 - name: Define default ALB listeners.
   ansible.builtin.set_fact:

roles/aws/aws_vpc_route/tasks/route.yml

@@ -7,6 +7,7 @@
       gateway_id: "{{ route.gateway_id | default(omit) }}"
       nat_gateway_id: "{{ route.nat_gateway_id | default(omit) }}"
       network_interface_id: "{{ route.network_interface_id | default(omit) }}"
+      transit_gateway_id: "{{ route.transit_gateway_id | default(omit) }}"
       vpc_peering_connection_id: "{{ route.vpc_peering_connection_id | default(omit) }}"
       instance_id: "{{ route.instance_id | default(omit) }}"
 

roles/debian/duplicity/README.md

@@ -0,0 +1,41 @@
+# Duplicity
+Role to install and configure [the Duplicity backup engine](https://duplicity.us/) for off site backups in Linux.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+duplicity:
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
+  backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
+  bucketname: "somebucket"
+  dirs:
+    - name: "/boot"
+      rules: []
+    - name: "/etc"
+      rules: []
+    - name: "/opt"
+      rules: []
+    - name: "/var"
+      rules:
+        - "+ /var/log/syslog*"
+        - "- /var"
+  exclude_other_filesystems: false
+  full_backup_frequency: "3M"
+  gpg_passphrase: "{{ lookup('password', _ce_provision_data_dir + '/' + inventory_hostname + '/duplicity-gpg-passphrase chars=ascii_letters,digits length=64') }}"
+  install_dir: "/opt/duplicity"
+  mail_recipient: "foo@bar.com"
+  retention_period: "12M"
+  # systemd timer settings
+  create_timer: true # sometimes you might want to trigger duplicity some other way than a systemd timer
+  on_calendar: "*-*-* 03:30:00" # see systemd.time documentation - https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html#Calendar%20Events
+
+```
+
+<!--ENDROLEVARS-->

roles/debian/duplicity/defaults/main.yml

@@ -1,8 +1,10 @@
 ---
 duplicity:
-  aws_access_key_id: "somekey"
-  aws_secret_access_key: "somesecret"
+  backend: s3 # currently also support b2 for Backblaze
+  access_key_id: "somekey"
+  secret_access_key: "somesecret"
   backend_url: "s3-eu-west-1.amazonaws.com"
+  s3_options: "--s3-european-buckets --s3-use-glacier-ir" # see the --s3 options in the documentation - https://duplicity.us/stable/duplicity.1.html#options
   bucketname: "somebucket"
   dirs:
     - name: "/boot"

roles/debian/duplicity/tasks/main.yml

@@ -28,23 +28,23 @@
 
 - name: Copy backup script in place.
   ansible.builtin.template:
-    src: duplicity_backup.j2
+    src: "duplicity_backup-{{ duplicity.backend }}.j2"
     dest: "{{ duplicity.install_dir }}/bin/duplicity_backup"
     owner: root
     group: root
     mode: 0700
 
 - name: Copy restore script in place.
   ansible.builtin.template:
-    src: duplicity_restore.j2
+    src: duplicity_restore-{{ duplicity.backend }}.j2
     dest: "{{ duplicity.install_dir }}/bin/duplicity_restore"
     owner: root
     group: root
     mode: 0700
 
 - name: Copy clean-up script in place.
   ansible.builtin.template:
-    src: duplicity_clean.j2
+    src: duplicity_clean-{{ duplicity.backend }}.j2
     dest: "{{ duplicity.install_dir }}/bin/duplicity_clean"
     owner: root
     group: root
@@ -69,7 +69,7 @@
   vars:
     timers:
       duplicity_backup:
-        timer_command: /usr/local/bin/duplicity_backup
+        timer_command: "{{ duplicity.install_dir }}/bin/duplicity_backup"
         timer_OnCalendar: "{{ duplicity.on_calendar }}"
   when: duplicity.create_timer
 

roles/debian/duplicity/templates/duplicity_backup.j2 → roles/debian/duplicity/templates/duplicity_backup-b2.j2

@@ -3,9 +3,9 @@
 # Duplicity Backup script
 
 # Declare and export secrets
-export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}}
-export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}}
-export PASSPHRASE={{duplicity.gpg_passphrase}}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+export B2_KEY_ID={{ duplicity.access_key_id }}
+export B2_SECRET_KEY={{ duplicity.secret_access_key }}
 
 if [ ! `whoami` = "root" ] ; then
   echo "You must run this script as root"
@@ -15,21 +15,21 @@ fi
 ## Configurable variables
 
 # How often should we make a full backup? Recommended: 3 months
-FULL_BACKUPS="{{duplicity.full_backup_frequency}}"
+FULL_BACKUPS="{{ duplicity.full_backup_frequency }}"
 
 # Remove old backups? 0 for no, 1 for yes
 REMOVE_OLD_BACKUPS=1
 
 # How often should we purge old backups? Recommended: 12 months.
-REMOVE_OLDER_THAN="{{duplicity.retention_period}}"
+REMOVE_OLDER_THAN="{{ duplicity.retention_period }}"
 
 # Args to pass to duplicity
 {% if duplicity.exclude_other_filesystems %}
-backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 --s3-use-new-style --s3-european-buckets"
+backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30"
 {% else %}
-backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 --s3-use-new-style --s3-european-buckets"
+backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30"
 {% endif %}
-maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force --s3-use-new-style --s3-european-buckets"
+maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force"
 
 # An array of directories to back up
 DIRS=(
@@ -46,23 +46,25 @@ for dir in ${DIRS[@]}; do
   echo "Backing up $dir..."
 
   extra_options=""
-  if [ -f "{{duplicity.install_dir}}/etc/$dir-include-exclude-filelist" ]; then
-    extra_options="--include-filelist {{duplicity.install_dir}}/etc/$dir-include-exclude-filelist"
+  if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then
+    extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist"
   fi
 
   # A special clause for /root. We don't want the local duplicity cache data
   if [ $dir = "/root" ]; then
     extra_options="$extra_options --exclude /root/.cache"
   fi
 
-  DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir
-  duplicity $backup_options $extra_options $dir $DEST || exit 1
+  DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir
+  /usr/local/bin/duplicity $backup_options $extra_options $dir $DEST || exit 1
 
   if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then
     # Do some maintenance on the remote end to clean up old backups
     echo "Performing routine maintenance on $dir..."
-    duplicity $maintenance_options $DEST || exit 1
+    /usr/local/bin/duplicity $maintenance_options $DEST || exit 1
   fi
 done
 
 unset PASSPHRASE
+unset B2_KEY_ID
+unset B2_SECRET_KEY

roles/debian/duplicity/templates/duplicity_backup-s3.j2

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Duplicity Backup script
+
+# Declare and export secrets
+export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }}
+export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+
+if [ ! `whoami` = "root" ] ; then
+  echo "You must run this script as root"
+  exit 1
+fi
+
+## Configurable variables
+
+# How often should we make a full backup? Recommended: 3 months
+FULL_BACKUPS="{{ duplicity.full_backup_frequency }}"
+
+# Remove old backups? 0 for no, 1 for yes
+REMOVE_OLD_BACKUPS=1
+
+# How often should we purge old backups? Recommended: 12 months.
+REMOVE_OLDER_THAN="{{ duplicity.retention_period }}"
+
+# Args to pass to duplicity
+{% if duplicity.exclude_other_filesystems %}
+backup_options="--full-if-older-than $FULL_BACKUPS --exclude-other-filesystems --num-retries=30 {{ duplicity.s3_options }}"
+{% else %}
+backup_options="--full-if-older-than $FULL_BACKUPS --num-retries=30 {{ duplicity.s3_options }}"
+{% endif %}
+maintenance_options="remove-older-than $REMOVE_OLDER_THAN --force {{ duplicity.s3_options }}"
+
+# An array of directories to back up
+DIRS=(
+{% for item in duplicity.dirs %}
+  {{ item.name }}
+{% endfor %}
+)
+
+
+## Backup code below. You should not need to edit anything here.
+
+# Loop over each dir and perform the backup.
+for dir in ${DIRS[@]}; do
+  echo "Backing up $dir..."
+
+  extra_options=""
+  if [ -f "{{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist" ]; then
+    extra_options="--include-filelist {{ duplicity.install_dir }}/etc/$dir-include-exclude-filelist"
+  fi
+
+  # A special clause for /root. We don't want the local duplicity cache data
+  if [ $dir = "/root" ]; then
+    extra_options="$extra_options --exclude /root/.cache"
+  fi
+
+  DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir
+  /usr/local/bin/duplicity $backup_options $extra_options $dir $DEST || exit 1
+
+  if [ $REMOVE_OLD_BACKUPS -eq 1 ]; then
+    # Do some maintenance on the remote end to clean up old backups
+    echo "Performing routine maintenance on $dir..."
+    /usr/local/bin/duplicity $maintenance_options $DEST || exit 1
+  fi
+done
+
+unset PASSPHRASE
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_ACCESS_KEY_ID

roles/debian/duplicity/templates/duplicity_clean.j2 → roles/debian/duplicity/templates/duplicity_clean-b2.j2

@@ -3,9 +3,9 @@
 # Duplicity Cleanup script
 
 # Declare and export secrets
-export AWS_ACCESS_KEY_ID={{duplicity.aws_access_key_id}}
-export AWS_SECRET_ACCESS_KEY={{duplicity.aws_secret_access_key}}
-export PASSPHRASE={{duplicity.gpg_passphrase}}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+export B2_KEY_ID={{ duplicity.access_key_id }}
+export B2_SECRET_KEY={{ duplicity.secret_access_key }}
 
 if [ ! `whoami` = "root" ] ; then
   echo "You must run this script as root"
@@ -15,7 +15,7 @@ fi
 ## Configurable variables
 
 # Args to pass to duplicity
-cleanup_options="clean --force --s3-use-new-style --s3-european-buckets"
+cleanup_options="clean --force"
 
 # An array of directories to clean
 DIRS=(
@@ -30,8 +30,10 @@ DIRS=(
 # Loop over each dir and perform the clean.
 for dir in ${DIRS[@]}; do
   echo "Cleaning up $dir..."
-  DEST=s3://{{duplicity.backend_url}}/{{duplicity.bucketname}}$dir
-  duplicity $cleanup_options $DEST || exit 1
+  DEST=b2://$B2_KEY_ID:$B2_SECRET_KEY@{{ duplicity.bucketname }}$dir
+  /usr/local/bin/duplicity $cleanup_options $DEST || exit 1
 done
 
 unset PASSPHRASE
+unset B2_KEY_ID
+unset B2_SECRET_KEY

roles/debian/duplicity/templates/duplicity_clean-s3.j2

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Duplicity Cleanup script
+
+# Declare and export secrets
+export AWS_ACCESS_KEY_ID={{ duplicity.access_key_id }}
+export AWS_SECRET_ACCESS_KEY={{ duplicity.secret_access_key }}
+export PASSPHRASE={{ duplicity.gpg_passphrase }}
+
+if [ ! `whoami` = "root" ] ; then
+  echo "You must run this script as root"
+  exit 1
+fi
+
+## Configurable variables
+
+# Args to pass to duplicity
+cleanup_options="clean --force {{ duplicity.s3_options }}"
+
+# An array of directories to clean
+DIRS=(
+{% for item in duplicity.dirs %}
+  {{ item.name }}
+{% endfor %}
+)
+
+
+## Cleanup code below. You should not need to edit anything here.
+
+# Loop over each dir and perform the clean.
+for dir in ${DIRS[@]}; do
+  echo "Cleaning up $dir..."
+  DEST=s3://{{ duplicity.backend_url }}/{{ duplicity.bucketname }}$dir
+   /usr/local/bin/duplicity $cleanup_options $DEST || exit 1
+done
+
+unset PASSPHRASE
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_ACCESS_KEY_ID