From 3c33e09cf9891e0e4b79e1af723df93b96393b31 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 5 Jul 2023 12:46:48 +0300 Subject: [PATCH 1/5] Automate the configuration of WAL-G cron jobs 1. If wal_g_install: true, setting up cron jobs for WAl-G. 2. Change the WAL-G config example to use with AWS S3 3. Move "cron_jobs" variable to vars/system.yml New variable "wal_g_cron_jobs" --- roles/wal-g/tasks/cron.yml | 16 ++++++++++++++ roles/wal-g/tasks/main.yml | 6 +++++ tags.md | 2 ++ vars/main.yml | 45 ++++++++++++++++++++++---------------- vars/system.yml | 21 ++++++++++++++++++ 5 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 roles/wal-g/tasks/cron.yml diff --git a/roles/wal-g/tasks/cron.yml b/roles/wal-g/tasks/cron.yml new file mode 100644 index 000000000..d618c9a89 --- /dev/null +++ b/roles/wal-g/tasks/cron.yml @@ -0,0 +1,16 @@ +--- +- name: Add WAL-G cron jobs + cron: + cron_file: "{{ item.file | default('') }}" + user: "{{ item.user | default('postgres') }}" + minute: "{{ item.minute | default('*') }}" + hour: "{{ item.hour | default('*') }}" + day: "{{ item.day | default('*') }}" + month: "{{ item.month | default('*') }}" + weekday: "{{ item.weekday | default('*') }}" + name: "{{ item.name }}" + disabled: "{{ item.disabled | default(False) }}" + state: "{{ item.state | default('present') }}" + job: "{{ item.job }}" + loop: "{{ wal_g_cron_jobs }}" + tags: wal_g_cron diff --git a/roles/wal-g/tasks/main.yml b/roles/wal-g/tasks/main.yml index dffcc38ff..47d09f0e3 100644 --- a/roles/wal-g/tasks/main.yml +++ b/roles/wal-g/tasks/main.yml @@ -259,4 +259,10 @@ mode: "0644" tags: wal-g, wal_g, wal_g_conf +- import_tasks: cron.yml + when: + - wal_g_cron_jobs is defined + - wal_g_cron_jobs | length > 0 + tags: wal-g, wal_g, wal_g_cron + ... diff --git a/tags.md b/tags.md index 4ad358eaf..42b7b8220 100644 --- a/tags.md +++ b/tags.md @@ -89,12 +89,14 @@ - wal_g - - wal_g_install - - wal_g_conf +- - wal_g_cron - pgbackrest - - pgbackrest_repo - - pgbackrest_install - - pgbackrest_conf - - pgbackrest_ssh_keys - - pgbackrest_stanza_create +- - pgbackrest_cron - pg_probackup - - pg_probackup_repo - - pg_probackup_install diff --git a/vars/main.yml b/vars/main.yml index e067f40d9..8b5b8910e 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -403,20 +403,38 @@ pg_probackup_patroni_cluster_bootstrap_command: "pg_probackup-{{ pg_probackup_ve wal_g_install: false # or 'true' wal_g_version: "2.0.1" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - - { option: "AWS_ACCESS_KEY_ID", value: "minio" } - - { option: "AWS_SECRET_ACCESS_KEY", value: "miniosecret" } - - { option: "AWS_ENDPOINT", value: "http://172.26.9.200:9000" } - - { option: "WALG_S3_PREFIX", value: "s3://bucket" } - - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } - - { option: "WALG_COMPRESSION_METHOD", value: "brotli" } + - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars + - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } # define values or pass via --extra-vars + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('') }}" } # define values or pass via --extra-vars + - { option: "WALG_COMPRESSION_METHOD", value: "brotli" } # or "lz4", "lzma", "zstd" - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir }}" } -# - { option: "AWS_REGION", value: "us-east-1" } -# - { option: "WALG_S3_CA_CERT_FILE", value: "/path/to/custom/ca/file" } +# - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } # to use Minio.io S3-compatible storage +# - { option: "AWS_ENDPOINT", value: "http://minio:9000" } # to use Minio.io S3-compatible storage # - { option: "", value: "" } wal_g_archive_command: "wal-g wal-push %p" wal_g_patroni_cluster_bootstrap_command: "wal-g backup-fetch {{ postgresql_data_dir }} LATEST" +wal_g_cron_jobs: + - name: "WAL-G: Create daily backup" + user: "postgres" + file: /etc/cron.d/walg + minute: "30" + hour: "3" + day: "*" + month: "*" + weekday: "*" + job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push" + - name: "WAL-G: Delete old backups" # older than 30 days (by default). Change according to your company's backup retention policy. + user: "postgres" + file: /etc/cron.d/walg + minute: "30" + hour: "6" + day: "*" + month: "*" + weekday: "*" + job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g delete before FIND_FULL $(date -d '-30 days' '+%FT%TZ') --confirm" + # pgBackRest pgbackrest_install: false # or 'true' pgbackrest_install_from_pgdg_repo: true # or 'false' @@ -499,17 +517,6 @@ pgbackrest_cron_jobs: job: "pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup" # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi" -cron_jobs: [] -# Example for walg -# - name: "WAL-G: Create daily backup" -# user: "postgres" -# file: /etc/cron.d/walg -# minute: "30" -# hour: "6" -# day: "*" -# month: "*" -# weekday: "*" -# job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push" # PITR mode (if patroni_cluster_bootstrap_method: "pgbackrest" or "wal-g"): # 1) The database cluster directory will be cleaned (for "wal-g") or overwritten (for "pgbackrest" --delta restore). diff --git a/vars/system.yml b/vars/system.yml index 87630da57..81de3ff2a 100644 --- a/vars/system.yml +++ b/vars/system.yml @@ -188,4 +188,25 @@ copy_files_to_all_server: [] # - { src: "files/ssl-cert-snakeoil.key", dest: "/etc/ssl/private/ssl-cert-snakeoil.key", owner: "postgres", group: "postgres", mode: "0600" } # - { src: "files/myfile", dest: "/path/to/myfile", owner: "postgres", group: "postgres", mode: "0640" } +# System cron jobs +cron_jobs: [] +# - name: "Example Job one" +# user: "postgres" +# file: /etc/cron.d/example_job_one +# minute: "00" +# hour: "1" +# day: "*" +# month: "*" +# weekday: "*" +# job: "echo 'example job one command'" +# - name: "Example Job two" +# user: "postgres" +# file: /etc/cron.d/example_job_two +# minute: "00" +# hour: "2" +# day: "*" +# month: "*" +# weekday: "*" +# job: "echo 'example job two command'" + ... From ff4d9e0b132f586b6e3c7265283d5b831e18f97e Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 5 Jul 2023 14:10:09 +0300 Subject: [PATCH 2/5] Update wal_g_json --- vars/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 8b5b8910e..e56cc476c 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -408,7 +408,10 @@ wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('') }}" } # define values or pass via --extra-vars - { option: "WALG_COMPRESSION_METHOD", value: "brotli" } # or "lz4", "lzma", "zstd" - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } - - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir }}" } + - { option: "PGHOST", value: "localhost" } + - { option: "PGPORT", value: "{{ postgresql_port }}" } + - { option: "PGUSER", value: "postgres" } + - { option: "PGDATABASE", value: "postgres" } # - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } # to use Minio.io S3-compatible storage # - { option: "AWS_ENDPOINT", value: "http://minio:9000" } # to use Minio.io S3-compatible storage # - { option: "", value: "" } From a87b2ae40d112689442393ecdd09010c2260a9fa Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 5 Jul 2023 15:55:49 +0300 Subject: [PATCH 3/5] Update wal_g_json PGUSER --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index e56cc476c..1611f8504 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -410,7 +410,7 @@ wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } - { option: "PGHOST", value: "localhost" } - { option: "PGPORT", value: "{{ postgresql_port }}" } - - { option: "PGUSER", value: "postgres" } + - { option: "PGUSER", value: "{{ patroni_superuser_username }}" } - { option: "PGDATABASE", value: "postgres" } # - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } # to use Minio.io S3-compatible storage # - { option: "AWS_ENDPOINT", value: "http://minio:9000" } # to use Minio.io S3-compatible storage From adb26908109bfaaaaade37640ca16855424beea7 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 5 Jul 2023 17:49:12 +0300 Subject: [PATCH 4/5] wal_g: use postgresql_unix_socket_dir --- vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/main.yml b/vars/main.yml index 1611f8504..8199134da 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -263,6 +263,7 @@ pending_restart: false # specify additional hosts that will be added to the pg_hba.conf postgresql_pg_hba: - { type: "local", database: "all", user: "{{ patroni_superuser_username }}", address: "", method: "trust" } + - { type: "local", database: "replication", user: "{{ patroni_superuser_username }}", address: "", method: "trust" } - { type: "local", database: "all", user: "all", address: "", method: "peer" } - { type: "host", database: "all", user: "all", address: "127.0.0.1/32", method: "{{ postgresql_password_encryption_algorithm }}" } - { type: "host", database: "all", user: "all", address: "::1/128", method: "{{ postgresql_password_encryption_algorithm }}" } @@ -408,10 +409,9 @@ wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('') }}" } # define values or pass via --extra-vars - { option: "WALG_COMPRESSION_METHOD", value: "brotli" } # or "lz4", "lzma", "zstd" - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } - - { option: "PGHOST", value: "localhost" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir }}" } - { option: "PGPORT", value: "{{ postgresql_port }}" } - { option: "PGUSER", value: "{{ patroni_superuser_username }}" } - - { option: "PGDATABASE", value: "postgres" } # - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } # to use Minio.io S3-compatible storage # - { option: "AWS_ENDPOINT", value: "http://minio:9000" } # to use Minio.io S3-compatible storage # - { option: "", value: "" } From 3e5aef878a095c90b12657cbf7966cb21ef47dcf Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Wed, 5 Jul 2023 18:06:54 +0300 Subject: [PATCH 5/5] Fix wal-g backup-push command --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 8199134da..04cec64da 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -427,7 +427,7 @@ wal_g_cron_jobs: day: "*" month: "*" weekday: "*" - job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push" + job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push {{ postgresql_data_dir }}" - name: "WAL-G: Delete old backups" # older than 30 days (by default). Change according to your company's backup retention policy. user: "postgres" file: /etc/cron.d/walg