Add the ability to configure a dedicated pgBackRest server (#379)

This commit introduces significant improvements to our backup and recovery capabilities with pgBackRest. The main feature is the ability to configure and manage the dedicated pgBackRest servers (pgbackrest_server_conf variable). Previously, backup configuration was limited to the database servers only. In addition to these improvements, add pgBackRest cron jobs via the "pgbackrest_cron_jobs" variable. Backup Jobs will be automatically configured. By default, the cron jobs is created on the database server. If 'pgbackrest_repo_host' is defined, the cron jobs will be created on the dedicated pgBackRest server. Updated the pgBackRest configuration template to enable configuration of a single dedicated pgBackRest server for multiple Postgres clusters.
vitabaks · Jun 12, 2023 · f826ca1 · f826ca1
1 parent b29734f
commit f826ca1
Show file tree

Hide file tree

Showing 10 changed files with 226 additions and 28 deletions.
diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -4,7 +4,7 @@
   gather_facts: true
 
   tasks:
-    - name: Set variables for molecule
+    - name: Set variables for Patroni cluster deployment test
       set_fact:
         firewall_enabled_at_boot: false
         firewall_enable_ipv6: false # Added to prevent test failures in CI.
@@ -21,5 +21,46 @@
         enable_timescale: true
       when: not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '<'))
 
+    - name: Clean yum cache (molecule containers)
+      command: yum clean all
+      when:
+        - ansible_os_family == "RedHat"
+        - ansible_distribution_major_version == '7'
+
+    - name: Clean dnf cache (molecule containers)
+      command: dnf clean all
+      when:
+        - ansible_os_family == "RedHat"
+        - ansible_distribution_major_version is version('8', '>=')
+
+    - name: Update apt cache (molecule containers)
+      apt:
+        update_cache: true
+        cache_valid_time: 3600
+      register: apt_status
+      until: apt_status is success
+      delay: 5
+      retries: 3
+      when: ansible_os_family == "Debian"
+
+    - name: Install openssh-server package (molecule containers)
+      become: true
+      package:
+        name: openssh-server
+        state: present
+
+    - name: Start ssh service (molecule containers)
+      become: true
+      systemd:
+        name: "{{ 'ssh' if ansible_os_family == 'Debian' else 'sshd' }}"
+        state: started
+        enabled: true
+
+    - name: Delete "/run/nologin" file (if exists)
+      become: true
+      file:
+        path: /run/nologin
+        state: absent
+
 - name: Deploy PostgreSQL Cluster
   import_playbook: ../../deploy_pgcluster.yml
diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
@@ -12,6 +12,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.0.20
     exposed_ports:
+      - 22/tcp
       - 2379/tcp # if dcs_type: "etcd"
       - 2380/tcp # if dcs_type: "etcd"
       - 8300/tcp # if dcs_type: "consul"
@@ -42,6 +43,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.0.21
     exposed_ports:
+      - 22/tcp
       - 2379/tcp
       - 2380/tcp
       - 8300/tcp
@@ -72,6 +74,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.0.22
     exposed_ports:
+      - 22/tcp
       - 2379/tcp
       - 2380/tcp
       - 8300/tcp

diff --git a/molecule/postgrespro/molecule.yml b/molecule/postgrespro/molecule.yml
@@ -12,6 +12,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.1.20
     exposed_ports:
+      - 22/tcp
       - 2379/tcp # if dcs_type: "etcd"
       - 2380/tcp # if dcs_type: "etcd"
       - 8300/tcp # if dcs_type: "consul"
@@ -42,6 +43,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.1.21
     exposed_ports:
+      - 22/tcp
       - 2379/tcp
       - 2380/tcp
       - 8300/tcp
@@ -72,6 +74,7 @@ platforms:
       - name: test_docker_network
         ipv4_address: 10.172.1.22
     exposed_ports:
+      - 22/tcp
       - 2379/tcp
       - 2380/tcp
       - 8300/tcp

diff --git a/roles/pgbackrest/stanza-create/tasks/main.yml b/roles/pgbackrest/stanza-create/tasks/main.yml
@@ -1,12 +1,11 @@
 ---
 
-- name: Get repo1-path value
-  set_fact:
-    repo1_path: "{{ pgbackrest_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}"
-  tags: pgbackrest, pgbackrest_stanza_create
-
 # Create a stanza locally (if "pgbackrest_repo_host" is not set)
 - block:
+    - name: Get repo1-path value
+      set_fact:
+        repo1_path: "{{ pgbackrest_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}"
+
     - name: "Make sure the {{ repo1_path }} directory exists"
       file:
         path: "{{ repo1_path }}"
@@ -34,6 +33,12 @@
     # It will create a stanza on the dedicated repository host only when pgbackrest_repo_host is defined and has a value.
     # The delegate_to parameter is used to execute the task on a different host than the one specified in the play's hosts parameter.
     # In this case, the task is delegated to the first host in the pgbackrest group in the invetory.
+    - name: Get repo1-path value
+      delegate_to: "{{ groups['pgbackrest'][0] }}"
+      run_once: true
+      set_fact:
+        repo1_path: "{{ pgbackrest_server_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}"
+
     - name: "Make sure the {{ repo1_path }} directory exists"
       delegate_to: "{{ groups['pgbackrest'][0] }}"
       run_once: true

diff --git a/roles/pgbackrest/tasks/cron.yml b/roles/pgbackrest/tasks/cron.yml
@@ -0,0 +1,38 @@
+---
+
+- name: Add pgbackrest cron jobs on database server
+  cron:
+    cron_file: "{{ item.file | default('') }}"
+    user: "{{ item.user | default('postgres')  }}"
+    minute: "{{ item.minute | default('*') }}"
+    hour: "{{ item.hour | default('*') }}"
+    day: "{{ item.day | default('*') }}"
+    month: "{{ item.month | default('*') }}"
+    weekday: "{{ item.weekday | default('*') }}"
+    name: "{{ item.name }}"
+    disabled: "{{ item.disabled | default(False) }}"
+    state: "{{ item.state | default('present') }}"
+    job: "{{ item.job }}"
+  loop: "{{ pgbackrest_cron_jobs }}"
+  when:
+    - "'postgres_cluster' in group_names"
+    - pgbackrest_repo_host | default('') | length < 1
+
+# Dedicated pgbackrest server (if "repo_host" is set)
+- name: Add pgbackrest cron jobs on pgbackrest server
+  cron:
+    cron_file: "{{ item.file | default('') }}"
+    user: "{{ item.user | default('postgres')  }}"
+    minute: "{{ item.minute | default('*') }}"
+    hour: "{{ item.hour | default('*') }}"
+    day: "{{ item.day | default('*') }}"
+    month: "{{ item.month | default('*') }}"
+    weekday: "{{ item.weekday | default('*') }}"
+    name: "{{ item.name }}"
+    disabled: "{{ item.disabled | default(False) }}"
+    state: "{{ item.state | default('present') }}"
+    job: "{{ item.job }}"
+  loop: "{{ pgbackrest_cron_jobs }}"
+  when:
+    - "'pgbackrest' in group_names"
+    - pgbackrest_repo_host | default('') | length > 0
diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml
@@ -123,6 +123,43 @@
   when: "'postgres_cluster' in group_names"
   tags: pgbackrest, pgbackrest_conf
 
+# Dedicated pgbackrest server (if "repo_host" is set)
+- block:
+    - name: Ensure config directory exist
+      file:
+        path: "{{ pgbackrest_conf_file | dirname }}"
+        state: directory
+        owner: "{{ pgbackrest_repo_user }}"
+        group: "{{ pgbackrest_repo_user }}"
+
+    - name: Ensure stanza config directory exist
+      file:
+        path: "{{ pgbackrest_conf_file | dirname }}/conf.d"
+        state: directory
+        owner: "{{ pgbackrest_repo_user }}"
+        group: "{{ pgbackrest_repo_user }}"
+
+    - name: "Generate global conf file {{ pgbackrest_conf_file }}"
+      template:
+        src: pgbackrest.server.conf.j2
+        dest: "{{ pgbackrest_conf_file }}"
+        owner: "{{ pgbackrest_repo_user }}"
+        group: "{{ pgbackrest_repo_user }}"
+        mode: "0644"
+
+    - name: "Generate stanza conf file {{ pgbackrest_conf_file | dirname }}/conf.d/{{ pgbackrest_stanza }}.conf"
+      template:
+        src: pgbackrest.server.stanza.conf.j2
+        dest: "{{ pgbackrest_conf_file | dirname }}/conf.d/{{ pgbackrest_stanza }}.conf"
+        owner: "{{ pgbackrest_repo_user }}"
+        group: "{{ pgbackrest_repo_user }}"
+        mode: "0644"
+  when:
+    - "'pgbackrest' in group_names"
+    - pgbackrest_repo_host is defined
+    - pgbackrest_repo_host | length > 0
+  tags: pgbackrest, pgbackrest_conf
+
 # if pgbackrest_repo_type: "posix" and pgbackrest_repo_host is set
 - import_tasks: ssh_keys.yml
   when:
@@ -131,6 +168,12 @@
     - pgbackrest_repo_host | length > 0
   tags: pgbackrest, pgbackrest_ssh_keys
 
+- import_tasks: cron.yml
+  when:
+    - pgbackrest_cron_jobs is defined
+    - pgbackrest_cron_jobs | length > 0
+  tags: pgbackrest, pgbackrest_cron
+
 # - import_tasks: bootstrap_script.yml
 #  when:
 #    - patroni_cluster_bootstrap_method is defined

diff --git a/roles/pgbackrest/tasks/ssh_keys.yml b/roles/pgbackrest/tasks/ssh_keys.yml
@@ -1,5 +1,27 @@
 ---
 
+- name: Ensure that the openssh-client package is installed
+  become: true
+  become_user: root
+  package:
+    name: openssh-client
+    state: present
+  when: ansible_os_family == "Debian"
+
+- name: Ensure that the openssh-clients package is installed
+  become: true
+  become_user: root
+  package:
+    name: openssh-clients
+    state: present
+  when: ansible_os_family == "RedHat"
+
+- name: Ensure "{{ pgbackrest_repo_user }}" exists on pgbackrest server
+  user:
+    name: "{{ pgbackrest_repo_user }}"
+    state: present
+  when: "'pgbackrest' in group_names"
+
 - name: ssh_keys | Ensure ssh key are created for "{{ pgbackrest_repo_user }}" user on pgbackrest server
   user:
     name: "{{ pgbackrest_repo_user }}"

diff --git a/roles/pgbackrest/templates/pgbackrest.server.conf.j2 b/roles/pgbackrest/templates/pgbackrest.server.conf.j2
@@ -0,0 +1,8 @@
+[global]
+{% for global in pgbackrest_server_conf.global %}
+{{ global.option }}={{ global.value }}
+{% endfor %}
+
+# Include stanzas configuration files
+include-path = {{ pgbackrest_conf_file | dirname }}/conf.d
+
diff --git a/roles/pgbackrest/templates/pgbackrest.server.stanza.conf.j2 b/roles/pgbackrest/templates/pgbackrest.server.stanza.conf.j2
@@ -0,0 +1,7 @@
+[{{ pgbackrest_stanza }}]
+{% for host in groups['postgres_cluster'] %}
+pg{{ loop.index }}-host={{ host }}
+pg{{ loop.index }}-port={{ postgresql_port }}
+pg{{ loop.index }}-path={{ postgresql_data_dir }}
+{% endfor %}
+
diff --git a/vars/main.yml b/vars/main.yml
@@ -415,8 +415,8 @@ pgbackrest_install: false  # or 'true'
 pgbackrest_install_from_pgdg_repo: true  # or 'false'
 pgbackrest_stanza: "{{ patroni_cluster_name }}"  # specify your --stanza
 pgbackrest_repo_type: "posix"  # or "s3", "gcs", "azure"
-pgbackrest_repo_host: ""  # dedicated repository host
-pgbackrest_repo_user: ""  # if "repo_host" is set
+pgbackrest_repo_host: ""  # dedicated repository host (optional)
+pgbackrest_repo_user: "postgres"
 pgbackrest_conf_file: "/etc/pgbackrest/pgbackrest.conf"
 # see more options https://pgbackrest.org/configuration.html
 pgbackrest_conf:
@@ -428,8 +428,8 @@ pgbackrest_conf:
     - { option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}" }
     - { option: "repo1-path", value: "/var/lib/pgbackrest" }
     - { option: "repo1-retention-full", value: "4" }
+    - { option: "repo1-retention-archive", value: "4" }
     - { option: "start-fast", value: "y" }
-    - { option: "delta", value: "y" }
     - { option: "stop-auto", value: "y" }
     - { option: "resume", value: "n" }
     - { option: "link-all", value: "y" }
@@ -440,32 +440,60 @@ pgbackrest_conf:
     - { option: "log-level-console", value: "info" }
     - { option: "process-max", value: "4" }
 #    - { option: "", value: "" }
+# (optional) dedicated backup server config (if "repo_host" is set)
+pgbackrest_server_conf:
+  global:
+    - { option: "log-level-file", value: "detail" }
+    - { option: "log-level-console", value: "info" }
+    - { option: "log-path", value: "/var/log/pgbackrest" }
+    - { option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}" }
+    - { option: "repo1-path", value: "/var/lib/pgbackrest" }
+    - { option: "repo1-retention-full", value: "4" }
+    - { option: "repo1-retention-archive", value: "4" }
+    - { option: "repo1-bundle", value: "y" }
+    - { option: "repo1-block", value: "y" }
+    - { option: "start-fast", value: "y" }
+    - { option: "stop-auto", value: "y" }
+    - { option: "resume", value: "n" }
+    - { option: "link-all", value: "y" }
+    - { option: "archive-check", value: "y" }
+    - { option: "archive-copy", value: "n" }
+    - { option: "backup-standby", value: "y" }
+#    - { option: "", value: "" }
+# the stanza section will be generated automatically
+
 pgbackrest_patroni_cluster_restore_command:
   '/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore'  # restore from latest backup
 #  '/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --type=time "--target=2020-06-01 11:00:00+03" --delta restore'  # Point-in-Time Recovery (example)
 
+# By default, the cron jobs is created on the database server.
+# If 'repo_host' is defined, the cron jobs will be created on the pgbackrest server.
+pgbackrest_cron_jobs:
+  - name: "pgBackRest: Full Backup"
+    file: /etc/cron.d/pgbackrest
+    user: "postgres"
+    minute: "30"
+    hour: "6"
+    day: "*"
+    month: "*"
+    weekday: "0"
+    job: "pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup"
+    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi"
+  - name: "pgBackRest: Diff Backup"
+    file: /etc/cron.d/pgbackrest
+    user: "postgres"
+    minute: "30"
+    hour: "6"
+    day: "*"
+    month: "*"
+    weekday: "1-6"
+    job: "pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup"
+    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi"
+
 cron_jobs: []
-#  - name: "pgBackRest: Full Backup"
-#    file: /etc/cron.d/pgbackrest
-#    user: "{{ patroni_superuser_username }}"
-#    minute: "30"
-#    hour: "6"
-#    day: "*"
-#    month: "*"
-#    weekday: "0"
-#    job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi"
-#  - name: "pgBackRest: Diff Backup"
-#    file: /etc/cron.d/pgbackrest
-#    user: "{{ patroni_superuser_username }}"
-#    minute: "30"
-#    hour: "6"
-#    day: "*"
-#    month: "*"
-#    weekday: "1-6"
-#    job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi"
 # Example for walg
 #  - name: "WAL-G: Create daily backup"
-#    user: "{{ patroni_superuser_username }}"
+#    user: "postgres"
 #    file: /etc/cron.d/walg
 #    minute: "30"
 #    hour: "6"