From 2d634f46eae68e0e1c5fe303240aefc03636706d Mon Sep 17 00:00:00 2001 From: tim Date: Mon, 20 Mar 2023 15:17:55 +0200 Subject: [PATCH 1/2] improving_old_builds_cleanup --- .../database_backup-mysql/defaults/main.yml | 2 ++ .../database_backup-mysql/tasks/cleanup-dump.yml | 2 +- .../database_backup-mysql/tasks/cleanup-rolling.yml | 3 +-- .../database_backup-mysql/tasks/cleanup.yml | 3 +-- roles/deploy_code/defaults/main.yml | 2 ++ roles/deploy_code/tasks/cleanup.yml | 10 +++++----- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index f7e070e9..ad824300 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -5,6 +5,8 @@ mysql_backup: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 + # Number of dumps/db to look up for cleanup. Note this is independent of databases/dump. + cleanup_history_depth: 50 # This can be one of the following: # - rotate: # Generates a new user/pwd pair per builds. diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index a9ea9ae2..1bbc5f53 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -4,5 +4,5 @@ ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index 2bdf29ee..ea4bbf7c 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -1,5 +1,4 @@ --- -# We assume it's safe to only go back 50 build back. - name: Delete mysql databases. community.mysql.mysql_db: name: "{{ database.database }}_{{ item }}" @@ -7,5 +6,5 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index ac71fcbb..b716212b 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -15,7 +15,6 @@ - ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" -# We assume it's safe to only go back 50 build back. - name: Delete mysql users. community.mysql.mysql_user: name: "{{ database.user }}_{{ item }}" @@ -24,6 +23,6 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} when: mysql_backup.credentials_handling == 'rotate' run_once: true diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index f9b84739..c5f2c47a 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -15,6 +15,8 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 + # Number of builds to look up for cleanup. Note this is independent of databases/dump. + cleanup_history_depth: 50 # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index a5da0d87..25c463fc 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -2,14 +2,14 @@ - name: Ensure codebase is writable. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -18,7 +18,7 @@ - name: Ensure permissions are set on builds directory. ansible.builtin.shell: cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -28,7 +28,7 @@ ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type != "squashfs" @@ -36,7 +36,7 @@ ansible.builtin.file: name: "{{ build_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type == "squashfs" From e960e1eb245b02408b1d578deab274d594206283 Mon Sep 17 00:00:00 2001 From: tim Date: Mon, 20 Mar 2023 17:13:11 +0200 Subject: [PATCH 2/2] improving_old_builds_cleanup_fix --- roles/_init/defaults/main.yml | 2 ++ .../database_backup-mysql/defaults/main.yml | 2 -- .../database_backup-mysql/tasks/cleanup-dump.yml | 2 +- .../database_backup-mysql/tasks/cleanup-rolling.yml | 2 +- .../database_backup-mysql/tasks/cleanup.yml | 2 +- roles/deploy_code/defaults/main.yml | 2 -- roles/deploy_code/tasks/cleanup.yml | 10 +++++----- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 00d03c05..971f1b9b 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -27,3 +27,5 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index ad824300..f7e070e9 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -5,8 +5,6 @@ mysql_backup: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 - # Number of dumps/db to look up for cleanup. Note this is independent of databases/dump. - cleanup_history_depth: 50 # This can be one of the following: # - rotate: # Generates a new user/pwd pair per builds. diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index 1bbc5f53..2609cabe 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -4,5 +4,5 @@ ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent - with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index ea4bbf7c..374e65ca 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -6,5 +6,5 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index b716212b..89cbf171 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -23,6 +23,6 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - mysql_backup.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} when: mysql_backup.credentials_handling == 'rotate' run_once: true diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index c5f2c47a..f9b84739 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -15,8 +15,6 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Number of builds to look up for cleanup. Note this is independent of databases/dump. - cleanup_history_depth: 50 # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 25c463fc..6f8dc1f6 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -2,14 +2,14 @@ - name: Ensure codebase is writable. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" - with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -18,7 +18,7 @@ - name: Ensure permissions are set on builds directory. ansible.builtin.shell: cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -28,7 +28,7 @@ ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type != "squashfs" @@ -36,7 +36,7 @@ ansible.builtin.file: name: "{{ build_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - deploy_code.cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type == "squashfs"