From 3ccfe51727c35f7cef25216f0ae4d0701d9dabb9 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Thu, 20 Jan 2022 10:13:40 +0000
Subject: [PATCH 001/139] Defer nightly backups, disable ASG processes during
 syncs and run syncs with backup (#94)

* Change location of nightly backup script and delegate the cron that runs it to the deploy server, if required (for ASGs).

* Set up nightly backup crons in separate files in /etc/cron.d

* Can't put site cron files in /etc/cron.d because the deploy user doesn't have perms.

* Try and add the ability to sync a site using a nightly backup instead of a fresh DB dump.

* Use Ansible modules to look up RDS host and to copy the nightly backup into place.

* Delegate PATH setup in db backup cron to localhost.

* Shell bad. Command good. But makes it convoluted. Oh well.

* Used wrong database name in source database copy.

* Try and disable the ReplaceUnhealthy auto scale process during syncs.

* Fix Drupal cron roles when deferring to deploy server.
---
 .../cron_database_backup-mysql/tasks/main.yml | 10 +++
 .../tasks/setup.yml                           | 18 ++++-
 roles/cron/cron_drupal7/tasks/main.yml        |  1 +
 roles/cron/cron_drupal8/tasks/main.yml        |  1 +
 .../database_sync-mysql/defaults/main.yml     |  9 +++
 .../database_sync-mysql/tasks/sync.yml        | 79 ++++++++++++++++++-
 6 files changed, 115 insertions(+), 3 deletions(-)

diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
index 04789df8..ffd14157 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
@@ -3,6 +3,16 @@
 # screwing the daily backup when using rolling db, we could
 # add a keep mechanism for backup scripts, like for the dumps themselves.
 # Nice to have more than anything.
+- name: Setup PATH in crontab.
+  cron:
+    name: PATH
+    env: true
+    job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
+  when:
+    - drupal.defer is defined
+    - drupal.defer
+
 - include_tasks: setup.yml
   vars:
     database: database
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
index bdc1534c..40b0c47e 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
@@ -25,12 +25,26 @@
 - name: Create backup script.
   template:
     src: "regular-backups.sh.j2"
-    dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+    dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
     mode: 0700
 
+- name: Define backup cron job command.
+  set_fact:
+    _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+
+- name: Define backup cron job command if deferred (ASG).
+  set_fact:
+    _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\""
+  when:
+    - drupal.defer is defined
+    - drupal.defer
+    - drupal.defer_target is defined
+    - drupal.defer_target | length > 0
+
 - name: Setup regular backup for MySQL.
   cron:
     name: "cron_mysql_{{ database.host }}_{{ database.original.database }}"
     minute: "{{ _cron_mysql_backup_minute }}"
     hour: "{{ _cron_mysql_backup_hour }}"
-    job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
+    job: "{{ _backup_cron_job_command }}"
+  delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml
index d0cbfce9..75fa05c7 100644
--- a/roles/cron/cron_drupal7/tasks/main.yml
+++ b/roles/cron/cron_drupal7/tasks/main.yml
@@ -4,6 +4,7 @@
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
   when:
     - drupal.defer is defined
     - drupal.defer
diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml
index d0cbfce9..75fa05c7 100644
--- a/roles/cron/cron_drupal8/tasks/main.yml
+++ b/roles/cron/cron_drupal8/tasks/main.yml
@@ -4,6 +4,7 @@
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
+  delegate_to: localhost
   when:
     - drupal.defer is defined
     - drupal.defer
diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index c1d7b259..48e3e4a0 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -15,6 +15,13 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
+        # Whether or not use to create a fresh database backup or use a nightly one.
+        fresh_db: true
+        # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default.
+        # This var is only used when fresh_db is set to "false".
+        dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
+        # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
@@ -25,3 +32,5 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuilddev
+        # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 81e9b2ce..b8182bc3 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -1,4 +1,40 @@
 ---
+- name: Get database source host region.
+  amazon.aws.ec2_metadata_facts:
+  register: mysql_sync_source_database_host_info
+  delegate_to: "{{ database.source.host }}"
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host.
+- name: Get database target host region.
+  amazon.aws.ec2_metadata_facts:
+  register: mysql_sync_target_database_host_info
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0
+
+- name: Disable ReplaceUnhealthy autoscale process on source ASG.
+  ansible.builtin.command: >
+    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+- name: Disable ReplaceUnhealthy autoscale process on target ASG.
+  ansible.builtin.command: >
+    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0
+
 - name: Register remote dump name (from database).
   set_fact:
     mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2"
@@ -33,7 +69,30 @@
 - name: Take a dump from source database.
   shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
-  when: not database.source.type == 'dump'
+  when:
+    - not database.source.type == 'dump'
+    - database.source.fresh_db
+
+- name: Find source database host.
+  ansible.builtin.command:
+    cmd: "grep 'host' {{ database.source.credentials_file }}"
+  register: mysql_host_info_grep
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
+
+- name: Register source database host.
+  set_fact:
+    mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}"
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
+
+- name: Copy a nightly backup for the source database.
+  ansible.builtin.copy:
+    src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}"
+    dest: "{{ mysql_sync_source_dump_path }}"
+    remote_src: true
+  delegate_to: "{{ database.source.host }}"
+  when: not database.source.fresh_db
 
 - name: Register tmp target dump name.
   set_fact:
@@ -95,3 +154,21 @@
     path: "{{ mysql_sync_target_dump_path }}"
     state: absent
   when: not database.target.type == 'dump'
+
+- name: Enable all autoscale processes on source ASG.
+  ansible.builtin.command: >
+    aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.source.asg is defined
+    - database.source.asg | length > 0
+    - database.source.fresh_db is defined
+    - database.source.fresh_db
+
+- name: Enable all autoscale processes on target ASG.
+  ansible.builtin.command: >
+    aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+  delegate_to: localhost
+  when:
+    - database.target.asg is defined
+    - database.target.asg | length > 0

From a54c09cb0ce5b6889e672ba8ea741b71a3da034b Mon Sep 17 00:00:00 2001
From: Dionisio <dionisiofernandez83@gmail.com>
Date: Thu, 10 Mar 2022 14:16:13 +0100
Subject: [PATCH 002/139] Added deploy.yml examples for Drupal 9 and Localgov.
 Updated tests (#97)

---
 .github/workflows/ce-deploy-test.yml        | 22 ++++++++
 ce-dev/ansible/examples/drupal9/deploy.yml  | 55 +++++++++++++++++++
 ce-dev/ansible/examples/localgov/deploy.yml | 59 +++++++++++++++++++++
 3 files changed, 136 insertions(+)
 create mode 100644 ce-dev/ansible/examples/drupal9/deploy.yml
 create mode 100644 ce-dev/ansible/examples/localgov/deploy.yml

diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml
index baa13347..141fc8e7 100644
--- a/.github/workflows/ce-deploy-test.yml
+++ b/.github/workflows/ce-deploy-test.yml
@@ -59,6 +59,28 @@ jobs:
           curl https://www.test.local
         shell: bash
 
+      - name: Run a test ce-dev deploy with Drupal 9
+        run: |
+          ce-dev create -p testnine -t drupal9 -d ~/testnine
+          cd ~/testnine
+          ce-dev init
+          ce-dev start
+          ce-dev provision
+          ce-dev deploy
+          curl https://www.testnine.local
+        shell: bash
+
+      - name: Run a test ce-dev deploy with Localgov
+        run: |
+          ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov
+          cd ~/testlocalgov
+          ce-dev init
+          ce-dev start
+          ce-dev provision
+          ce-dev deploy
+          curl https://www.testlocalgov.local
+        shell: bash
+
       # Builds the table of contents for the docs
       - name: Documentation (build table of contents)
         if: ${{ github.event.pull_request.base.ref == '1.x' }}
diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml
new file mode 100644
index 00000000..deeef9ee
--- /dev/null
+++ b/ce-dev/ansible/examples/drupal9/deploy.yml
@@ -0,0 +1,55 @@
+---
+# Template playbook for a local Drupal9 codebase.
+- hosts: deploy-web
+  vars:
+    - project_name: example
+    - project_type: drupal8
+    - webroot: web
+    - build_type: local
+    - _env_type: dev
+    - _domain_name: www.{{ project_name }}.local
+    # Path to your project root. This must match the "volume" set in the docker-compose template.
+    - deploy_path: /home/ce-dev/deploy/live.local
+    # This actually does not take any backup, but is needed to populate settings.php.
+    - mysql_backup:
+        handling: none
+        credentials_handling: static
+    # A list of Drupal sites (for multisites).
+    - drupal:
+        sites:
+          - folder: "default"
+            public_files: "sites/default/files"
+            install_command: "-y si"
+            # Toggle config import on/off. Disabled for initial passes.
+            config_import_command: ""
+            # config_import_command: "cim"
+            config_sync_directory: "config/sync"
+            sanitize_command: "sql-sanitize"
+            # Remove after initial pass, to avoid reinstalling Drupal.
+            force_install: yes
+            base_url: "https://{{ _domain_name }}"
+    # Composer command to run.
+    - composer:
+        command: install
+        no_dev: no
+        working_dir: "{{ deploy_path }}"
+        apcu_autoloader: no
+  pre_tasks:
+    # You can safely remove these steps once you have a working composer.json.
+    - name: Download composer file.
+      get_url:
+        url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json
+        dest: "{{ deploy_path }}/composer.json"
+        force: no
+    - name: Install drush.
+      command:
+        cmd: composer require drush/drush:11.*
+        chdir: "{{ deploy_path }}"
+  roles:
+    - _init # Sets some variables the deploy scripts rely on.
+    - composer # Composer install step.
+    - database_backup # This is still needed to generate credentials.
+    - config_generate # Generates settings.php
+    #    - sync/database_sync # Grab database from a remote server.
+    - database_apply # Run drush updb and config import.
+    - _exit # Some common housekeeping.
\ No newline at end of file
diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml
new file mode 100644
index 00000000..9844206b
--- /dev/null
+++ b/ce-dev/ansible/examples/localgov/deploy.yml
@@ -0,0 +1,59 @@
+---
+# Template playbook for a local localgov codebase.
+- hosts: deploy-web
+  vars:
+    - project_name: example
+    - project_type: drupal8
+    - webroot: web
+    - build_type: local
+    - _env_type: dev
+    - _domain_name: www.{{ project_name }}.local
+    # Path to your project root. This must match the "volume" set in the docker-compose template.
+    - deploy_path: /home/ce-dev/deploy/live.local
+    # This actually does not take any backup, but is needed to populate settings.php.
+    - mysql_backup:
+        handling: none
+        credentials_handling: static
+    # A list of Drupal sites (for multisites).
+    - drupal:
+        sites:
+          - folder: "default"
+            public_files: "sites/default/files"
+            install_command: "-y si localgov"
+            # Toggle config import on/off. Disabled for initial passes.
+            config_import_command: ""
+            # config_import_command: "cim"
+            config_sync_directory: "config/sync"
+            sanitize_command: "sql-sanitize"
+            # Remove after initial pass, to avoid reinstalling Drupal.
+            force_install: yes
+            base_url: "https://{{ _domain_name }}"
+    # Composer command to run.
+    - composer:
+        command: install
+        no_dev: no
+        working_dir: "{{ deploy_path }}"
+        apcu_autoloader: no
+  pre_tasks:
+    # You can safely remove these steps once you have a working composer.json.
+    - name: Download composer file.
+        get_url:
+          url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json
+          dest: "{{ deploy_path }}/composer.json"
+          force: false
+    - name: Install drush.
+        command:
+          cmd: composer require drush/drush:11.*
+          chdir: "{{ deploy_path }}"
+    - name: Install localgov.
+        command:
+          cmd: composer require localgovdrupal/localgov
+          chdir: "{{ deploy_path }}"
+  roles:
+    - _init # Sets some variables the deploy scripts rely on.
+    - composer # Composer install step.
+    - database_backup # This is still needed to generate credentials.
+    - config_generate # Generates settings.php
+    #    - sync/database_sync # Grab database from a remote server.
+    - database_apply # Run drush updb and config import.
+    - _exit # Some common housekeeping.
\ No newline at end of file

From 223a3b5ecec42997d9e52de690cf7b8c5f9f19ab Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 31 Mar 2022 12:43:36 +0200
Subject: [PATCH 003/139] Adding a new SimpleSAMLphp meta role. (#100)

---
 roles/_meta/deploy-simplesamlphp/README.md      |  9 +++++++++
 roles/_meta/deploy-simplesamlphp/tasks/main.yml | 17 +++++++++++++++++
 .../deploy_code-simplesamlphp/tasks/main.yml    |  1 +
 3 files changed, 27 insertions(+)
 create mode 100644 roles/_meta/deploy-simplesamlphp/README.md
 create mode 100644 roles/_meta/deploy-simplesamlphp/tasks/main.yml
 create mode 100644 roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml

diff --git a/roles/_meta/deploy-simplesamlphp/README.md b/roles/_meta/deploy-simplesamlphp/README.md
new file mode 100644
index 00000000..3d82e671
--- /dev/null
+++ b/roles/_meta/deploy-simplesamlphp/README.md
@@ -0,0 +1,9 @@
+# SimpleSAMLphp
+Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer.
+
+This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision:
+
+* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/roles/_meta/deploy-simplesamlphp/tasks/main.yml b/roles/_meta/deploy-simplesamlphp/tasks/main.yml
new file mode 100644
index 00000000..7af28fa2
--- /dev/null
+++ b/roles/_meta/deploy-simplesamlphp/tasks/main.yml
@@ -0,0 +1,17 @@
+---
+# Default SimpleSAMLphp role. This is suitable for a standalone SimpleSAMLphp installation
+
+- ansible.builtin.import_role:
+    name: _init
+- ansible.builtin.import_role:
+    name: deploy_code
+- ansible.builtin.import_role:
+    name: composer
+- ansible.builtin.import_role:
+    name: database_backup
+- ansible.builtin.import_role:
+    name: live_symlink
+- ansible.builtin.import_role:
+    name: cache_clear/cache_clear-opcache
+- ansible.builtin.import_role:
+    name: _exit
diff --git a/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml
new file mode 100644
index 00000000..03c03856
--- /dev/null
+++ b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml
@@ -0,0 +1 @@
+# Nothing to do.
\ No newline at end of file

From 32843d53a625b68bc3a3f7a8cfcced5c90df3f49 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 13 Apr 2022 13:09:01 +0200
Subject: [PATCH 004/139] Allowing users to set cachetool version properly.
 (#102)

---
 roles/cli/cachetool/defaults/main.yml |  2 +-
 roles/cli/cachetool/tasks/main.yml    | 13 ++++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml
index ad2c6711..5ec1b2d7 100644
--- a/roles/cli/cachetool/defaults/main.yml
+++ b/roles/cli/cachetool/defaults/main.yml
@@ -1,3 +1,3 @@
 ---
 cachetool:
-  version: 8.2.2
\ No newline at end of file
+  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
\ No newline at end of file
diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index 5a1969c6..aa27df04 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -13,7 +13,7 @@
   when:
     - deploy_operation == 'deploy'
 
-- name: Download cachetool installer.
+- name: Download latest cachetool installer.
   get_url:
     url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar"
     dest: "{{ cachetool_bin }}"
@@ -21,3 +21,14 @@
   when:
     - deploy_operation == 'deploy'
     - not cachetool_global.stat.exists
+    - cachetool.version == 'latest'
+
+- name: "Download cachetool version {{ cachetool.version }} installer."
+  get_url:
+    url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
+    dest: "{{ cachetool_bin }}"
+    mode: 0755
+  when:
+    - deploy_operation == 'deploy'
+    - not cachetool_global.stat.exists
+    - cachetool.version != 'latest'

From 91ea64efc91984ce88138857898b927b5ffd7b25 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 14 Apr 2022 17:13:09 +0200
Subject: [PATCH 005/139] Deploy ami pr 1.x (#106)

* Including an ami.yml for packing AMIs on build finale.

* New api_call role, focused on GitLab for now.

* Optionally trigger an infra build with an API call.
---
 roles/api_call/README.md            |  9 ++++++++
 roles/api_call/defaults/main.yml    | 13 +++++++++++
 roles/api_call/tasks/main.yml       | 35 +++++++++++++++++++++++++++++
 roles/deploy_code/defaults/main.yml | 21 +++++++++++++++++
 roles/deploy_code/tasks/cleanup.yml | 27 +++++++++++++++++-----
 roles/deploy_code/tasks/deploy.yml  | 10 ++++-----
 roles/deploy_code/tasks/main.yml    |  2 +-
 7 files changed, 105 insertions(+), 12 deletions(-)
 create mode 100644 roles/api_call/README.md
 create mode 100644 roles/api_call/defaults/main.yml
 create mode 100644 roles/api_call/tasks/main.yml

diff --git a/roles/api_call/README.md b/roles/api_call/README.md
new file mode 100644
index 00000000..83d1ab28
--- /dev/null
+++ b/roles/api_call/README.md
@@ -0,0 +1,9 @@
+# API call
+
+Making RESTful API calls to other platforms.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/roles/api_call/defaults/main.yml b/roles/api_call/defaults/main.yml
new file mode 100644
index 00000000..091342e8
--- /dev/null
+++ b/roles/api_call/defaults/main.yml
@@ -0,0 +1,13 @@
+---
+api_call:
+  type: gitlab
+  base_url: https://gitlab.example.com/api/v4/
+  path: projects # see documentation - https://docs.gitlab.com/ee/api/
+  method: GET
+  token: "" # empty means anonymous action
+  token_type: trigger # options are 'trigger' or 'personal'
+  variables: []
+  status_codes:
+    - 200
+    - 201
+    - 202
\ No newline at end of file
diff --git a/roles/api_call/tasks/main.yml b/roles/api_call/tasks/main.yml
new file mode 100644
index 00000000..c837df2d
--- /dev/null
+++ b/roles/api_call/tasks/main.yml
@@ -0,0 +1,35 @@
+---
+- name: Ensure variables are empty.
+  ansible.builtin.set_fact:
+    _api_call_variables: ""
+    _api_call_url: ""
+
+- name: Build HTML escaped variable string.
+  ansible.builtin.set_fact:
+    _api_call_variables: "{{ _api_call_variables + ('' if ansible_loop.first else '&') + 'variables' + item }}"
+  with_items: "{{ api_call.variables }}"
+  loop_control:
+    extended: true
+  when: api_call.variables | length > 0
+
+- name: Build anonymous API call URL.
+  ansible.builtin.set_fact:
+    _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{{ _api_call_variables }}"
+  when: api_call.token | length == 0
+
+- name: Build token authenticated API call URL.
+  ansible.builtin.set_fact:
+    _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{% if api_call.token_type == 'trigger' %}token={% else %}private_token={% endif %}{{ api_call.token }}&{{ _api_call_variables }}"
+  when: api_call.token | length > 0
+
+- name: Display URL to call.
+  ansible.builtin.debug:
+    msg: "{{ _api_call_url }}"
+
+- name: Make API call.
+  ansible.builtin.uri:
+    url: "{{ _api_call_url }}"
+    method: "{{ api_call.method }}"
+    return_content: true
+    status_code: "{{ api_call.status_codes }}"
+  register: _api_call_return
diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index 19e45eee..d616bf08 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -21,3 +21,24 @@ deploy_code:
   # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash.
   perms_fix_path: ""
   # perms_fix_path: "www/sites/default"
+  # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
+  rebuild_infra: false
+  # Details of API call to trigger. See api_call role.
+  api_call:
+    type: gitlab
+    base_url: https://gitlab.example.com/api/v4/
+    path: projects/1/ref/main/trigger/pipeline
+    method: POST
+    token: asdf-1234
+    token_type: trigger
+    variables: []
+    # example build parameters
+    #  - "[ENV]=dev"
+    #  - "[PLAY]=myserver.yml"
+    #  - "[RESOURCE]=myserver-example-com"
+    #  - "[REGION]=eu-west-1"
+    #  - "[EXTRA_PARAMS]=--force"
+    status_codes:
+      - 200
+      - 201
+      - 202
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index dc72c752..4e648d25 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -1,13 +1,13 @@
 ---
 - name: Ensure codebase is writable.
-  shell:
+  ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi"
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   become: true
   when: "www_user != deploy_user"
 
 - name: Ensure permissions are set on directory.
-  shell:
+  ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
@@ -15,13 +15,13 @@
     - deploy_code.perms_fix_path | length > 1
 
 - name: Delete codebases.
-  file:
+  ansible.builtin.file:
     name: "{{ deploy_path_prefix }}{{ item }}"
     state: absent
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
 
 - name: Create a tarball of the deployed codebases.
-  command:
+  ansible.builtin.command:
     cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}"
   when:
     - deploy_code.mount_sync is defined
@@ -29,7 +29,7 @@
   run_once: true
 
 - name: Create destination folder.
-  file:
+  ansible.builtin.file:
     path: "{{ deploy_code.mount_sync }}"
     state: directory
     mode: "0755"
@@ -39,9 +39,24 @@
   run_once: true
 
 - name: Move to final destination.
-  command:
+  ansible.builtin.command:
     cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
   run_once: true
+
+- name: Trigger an infrastructure rebuild.
+  ansible.builtin.include_role:
+    name: api_call
+  vars:
+    api_call:
+      type: "{{ deploy_code.api_call.type }}"
+      base_url: "{{ deploy_code.api_call.base_url }}"
+      path: "{{ deploy_code.api_call.path }}"
+      method: "{{ deploy_code.api_call.method }}"
+      token: "{{ deploy_code.api_call.token }}"
+      token_type: "{{ deploy_code.api_call.token_type }}"
+      variables: "{{ deploy_code.api_call.variables }}"
+      status_codes: "{{ deploy_code.api_call.status_codes }}"
+  when: deploy_code.rebuild_infra
diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml
index 3693eb70..0d15425c 100644
--- a/roles/deploy_code/tasks/deploy.yml
+++ b/roles/deploy_code/tasks/deploy.yml
@@ -1,6 +1,6 @@
 ---
 - name: Copy project repository.
-  synchronize:
+  ansible.posix.synchronize:
     src: "{{ _ce_deploy_build_dir }}/"
     dest: "{{ deploy_path }}"
     archive: true
@@ -8,17 +8,17 @@
       - "--exclude=.git"
 
 - name: Ensure project repository is readable.
-  file:
+  ansible.builtin.file:
     path: "{{ deploy_path }}"
     state: directory
     mode: 0755
 
 - name: Project specific tasks.
-  include_role:
+  ansible.builtin.include_role:
     name: "deploy_code/deploy_code-{{ project_type }}"
 
 - name: Generate additional templates.
-  template:
+  ansible.builtin.template:
     src: "{{ template.src }}"
     dest: "{{ deploy_path }}/{{ template.dest }}"
   with_items: "{{ deploy_code.templates }}"
@@ -29,7 +29,7 @@
     - deploy_operation == 'deploy'
 
 - name: Create additional symlinks.
-  file:
+  ansible.builtin.file:
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
diff --git a/roles/deploy_code/tasks/main.yml b/roles/deploy_code/tasks/main.yml
index a8021dcf..f4bd05af 100644
--- a/roles/deploy_code/tasks/main.yml
+++ b/roles/deploy_code/tasks/main.yml
@@ -1,2 +1,2 @@
 ---
-- include_tasks: "{{ deploy_operation }}.yml"
+- ansible.builtin.include_tasks: "{{ deploy_operation }}.yml"

From 371b6b9d8b28a908facfe2d80779efd76cb64b40 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 20 Apr 2022 16:20:36 +0200
Subject: [PATCH 006/139] Making the MySQL dump command for routine back-ups
 less aggressive. (#104)

* Making the MySQL dump command for routine back-ups less aggressive.

* Making max_allowed_packets a variable we can set.

* Adding dump command to a re-usable central var.

* Renaming _init var so it's easier to distinguish in the code.
---
 roles/_init/defaults/main.yml                                | 1 +
 .../cron_database_backup-mysql/defaults/main.yml             | 1 +
 .../templates/regular-backups.sh.j2                          | 5 +++--
 .../database_backup/database_backup-mysql/defaults/main.yml  | 1 +
 .../database_backup-mysql/tasks/deploy-dump.yml              | 2 +-
 .../database_backup-mysql/tasks/deploy-rolling.yml           | 2 +-
 .../sync/database_sync/database_sync-mysql/defaults/main.yml | 1 +
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml  | 2 +-
 8 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 06470a09..9821f131 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -2,6 +2,7 @@
 # Common defaults. Given the "_init" role is mandatory,
 # this will ensure defaults to other roles too.
 deploy_user: "deploy"
+_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
index c6c57269..29b8003b 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
@@ -2,3 +2,4 @@
 cron_mysql_backup:
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
   keep: 10
+  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
index 901cfcf7..fb60cdea 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
@@ -10,7 +10,8 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2"
 KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1))
 
 backup(){
-  mysqldump -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL"
+  {{ cron_mysql_backup.mysqldump_command }} \
+	    -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL"
   ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME"
 }
 
@@ -23,4 +24,4 @@ cleanup(){
 }
 
 backup
-cleanup
\ No newline at end of file
+cleanup
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 3ad97b6c..09f44ca3 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -2,6 +2,7 @@
 mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
+  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 4b470b89..57a46e2e 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -15,6 +15,6 @@
   run_once: true
 
 - name: Take a database dump.
-  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index fe32089a..cbade053 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -12,6 +12,6 @@
   run_once: true
 
 - name: Populate new database.
-  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
+  shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 48e3e4a0..008f0cf4 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -1,5 +1,6 @@
 ---
 mysql_sync:
+  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index b8182bc3..778092bc 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -67,7 +67,7 @@
   when: not database.source.type == 'rolling'
 
 - name: Take a dump from source database.
-  shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
   when:
     - not database.source.type == 'dump'

From a9cc7adf64a8c7959f971df84df9e6172d0cf4c6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 20 Apr 2022 18:18:47 +0200
Subject: [PATCH 007/139] Fix database backups pr 1.x (#109)

* Making the MySQL dump command for routine back-ups less aggressive.

* Making max_allowed_packets a variable we can set.

* Adding dump command to a re-usable central var.

* Renaming _init var so it's easier to distinguish in the code.

* Defaults file must be the first param for mysqldump.
---
 roles/_init/defaults/main.yml                                   | 2 +-
 .../cron_database_backup-mysql/defaults/main.yml                | 2 +-
 .../cron_database_backup-mysql/templates/regular-backups.sh.j2  | 2 +-
 roles/database_backup/database_backup-mysql/defaults/main.yml   | 2 +-
 .../database_backup/database_backup-mysql/tasks/deploy-dump.yml | 2 +-
 .../database_backup-mysql/tasks/deploy-rolling.yml              | 2 +-
 roles/sync/database_sync/database_sync-mysql/defaults/main.yml  | 2 +-
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml     | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 9821f131..c4d15b78 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -2,7 +2,7 @@
 # Common defaults. Given the "_init" role is mandatory,
 # this will ensure defaults to other roles too.
 deploy_user: "deploy"
-_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
index 29b8003b..f0c869f3 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml
@@ -2,4 +2,4 @@
 cron_mysql_backup:
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
   keep: 10
-  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
index fb60cdea..33c27a5a 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
@@ -10,7 +10,7 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2"
 KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1))
 
 backup(){
-  {{ cron_mysql_backup.mysqldump_command }} \
+  mysqldump {{ cron_mysql_backup.mysqldump_params }} \
 	    -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL"
   ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME"
 }
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 09f44ca3..f7e070e9 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -2,7 +2,7 @@
 mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
-  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 57a46e2e..d06b1543 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -15,6 +15,6 @@
   run_once: true
 
 - name: Take a database dump.
-  shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index cbade053..1db841b0 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -12,6 +12,6 @@
   run_once: true
 
 - name: Populate new database.
-  shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
+  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 008f0cf4..0ab99e98 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -1,6 +1,6 @@
 ---
 mysql_sync:
-  mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 778092bc..c3ff0b9e 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -67,7 +67,7 @@
   when: not database.source.type == 'rolling'
 
 - name: Take a dump from source database.
-  shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
   when:
     - not database.source.type == 'dump'

From 4c4c3ad1f9ecd25e25e9c2e5f79d4cb8108c9f34 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Fri, 22 Apr 2022 10:28:11 +0100
Subject: [PATCH 008/139] Fix MySQL backup deferral. (#110)

---
 .../cron_database_backup-mysql/tasks/setup.yml                  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
index 40b0c47e..d09fbbac 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
@@ -47,4 +47,4 @@
     minute: "{{ _cron_mysql_backup_minute }}"
     hour: "{{ _cron_mysql_backup_hour }}"
     job: "{{ _backup_cron_job_command }}"
-  delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
+  delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}"

From 029848d1163191087bd85f372973594b283aad17 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Tue, 26 Apr 2022 11:30:31 +0100
Subject: [PATCH 009/139] Files recurse fix pr 1.x (#112)

* Don't recurse through site directory when setting permissions during config_generate step.

* Update drupal7 config_generate perms update task and use true/false instead of yes/no.
---
 .../config_generate/config_generate-drupal7/tasks/settings.yml | 3 +--
 .../config_generate/config_generate-drupal8/tasks/settings.yml | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
index f82c9aa7..277899a0 100644
--- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
@@ -15,9 +15,8 @@
   file:
     path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
     state: directory
-    recurse: true
     mode: 0775
-  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become: "{{ false if www_user == deploy_user else true }}"
 
 - name: Generates settings.php file.
   template:
diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index a9937d8e..70d9b113 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -19,9 +19,8 @@
   file:
     path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
     state: directory
-    recurse: true
     mode: 0775
-  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become: "{{ false if www_user == deploy_user else true }}"
 
 - name: Generates settings.php file.
   template:

From b71fa83d6b4deda1daa4532c3a00e3835118cf4a Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Fri, 29 Apr 2022 11:52:28 +0100
Subject: [PATCH 010/139] Improve multisite support (#115)

* Pass -l option in drush commands to specify site name, which should match the site folder name.

* Move -l option in drush commands to the first option instead of the last.
---
 roles/cache_clear/cache_clear-drupal7/tasks/main.yml        | 2 +-
 roles/cache_clear/cache_clear-drupal8/tasks/main.yml        | 2 +-
 .../database_apply-drupal7/tasks/features.yml               | 4 ++--
 roles/database_apply/database_apply-drupal7/tasks/main.yml  | 4 ++--
 roles/database_apply/database_apply-drupal8/tasks/main.yml  | 6 +++---
 .../maintenance_mode-drupal-core/tasks/offline.yml          | 4 ++--
 .../maintenance_mode-drupal-core/tasks/online.yml           | 4 ++--
 .../admin_creds/admin_creds-drupal7/tasks/admin.yml         | 4 ++--
 .../admin_creds/admin_creds-drupal8/tasks/admin.yml         | 4 ++--
 roles/sanitize/sanitize-drupal8/tasks/main.yml              | 2 +-
 10 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
index 1261ff4b..6f72b3a5 100644
--- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: Clear Drupal 7 cache.
   command:
-    cmd: "{{ drush_bin }} -y cc all"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index cfd7fdd3..5b780086 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: Clear Drupal cache.
   command:
-    cmd: "{{ drush_bin }} -y cr"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml
index 19d58e9b..edba46bf 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/features.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml
@@ -1,8 +1,8 @@
 ---
 - name: Check if Features module is enabled.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} pm-info features | grep ':  enabled' | wc -l"
+  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ':  enabled' | wc -l"
   register: features_enabled
 
 - name: Revert Drupal configuration from Features.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -y {{ site.revert_features_command }}"
+  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}"
   when: features_enabled.stdout == "1"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 048a1457..7c2201e2 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -2,7 +2,7 @@
 
 - name: Install Drupal.
   shell:
-    cmd: "{{ drush_bin }} -y si"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y si"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -21,7 +21,7 @@
 
 - name: Apply Drupal database updates.
   shell:
-    cmd: "{{ drush_bin }} -y updb"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   with_items: "{{ drupal.sites }}"
   loop_control:
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 5634faca..dc138585 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -15,7 +15,7 @@
 
 - name: Install Drupal.
   command:
-    cmd: "{{ drush_bin }} {{ site.install_command }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -34,7 +34,7 @@
 
 - name: Apply Drupal database updates.
   command:
-    cmd: "{{ drush_bin }} -y updb"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -44,7 +44,7 @@
 
 - name: Import configuration.
   command:
-    cmd: "{{ drush_bin }} -y {{ site.config_import_command }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index 2872226a..cbe65b07 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -2,7 +2,7 @@
 # @todo this needs refactoring for multisite.
 - name: Enable maintenance mode.
   command:
-    cmd: "{{ drush_bin }} state:set system.maintenance_mode 1 --input-format=integer"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -12,7 +12,7 @@
 
 - name: Enable maintenance mode D7.
   shell:
-    cmd: "{{ drush_bin }} vset maintenance_mode 1"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 99057084..e6a9cbf5 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -1,7 +1,7 @@
 ---
 - name: Disable maintenance mode.
   command:
-    cmd: "{{ drush_bin }} state:set system.maintenance_mode 0 --input-format=integer"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -10,7 +10,7 @@
 
 - name: Disable maintenance mode D7.
   shell:
-    cmd: "{{ drush_bin }} vset maintenance_mode 0"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
index 98401c5c..c8d75bd7 100644
--- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
+++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
@@ -6,6 +6,6 @@
     _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}"
 
 - name: Reset admin username.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\""
+  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\""
 - name: Reset admin password.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'"
\ No newline at end of file
+  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'"
\ No newline at end of file
diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
index 0302308b..c2adafdf 100644
--- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
+++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
@@ -4,14 +4,14 @@
 # Loading the user directly is akward, but at least means we don't bypass entity update.
 - name: Reset admin username.
   command:
-    cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' "
+    cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' "
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when:
     - admin_creds.username | length > 1
 
 - name: Reset admin password.
   command:
-    cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' "
+    cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' "
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when:
     - admin_creds.password | length > 1
diff --git a/roles/sanitize/sanitize-drupal8/tasks/main.yml b/roles/sanitize/sanitize-drupal8/tasks/main.yml
index 0a33f5a8..26675f27 100644
--- a/roles/sanitize/sanitize-drupal8/tasks/main.yml
+++ b/roles/sanitize/sanitize-drupal8/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: Sanitize database.
   command:
-    cmd: "{{ drush_bin }} -y {{ site.sanitize_command }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.sanitize_command }}"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"

From c933dc8224e42106df909e29cdb8928f5c5da580 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Mon, 23 May 2022 16:19:31 +0100
Subject: [PATCH 011/139] Static credentials handling fix pr 1.x (#119)

* Attempt to fix static credentials handling when deploying to an ASG.

* Include build_type in static password file location.

* Move to using new location for static credentials handling, but try to catch any legacy stuff too, for now.

* Add a couple of comments to legacy handling of static credentials.
---
 .../database_backup-mysql/tasks/deploy.yml    | 49 ++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index a67d240e..dbb39de7 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -24,8 +24,55 @@
 - set_fact:
     _mysql_build_user_name: "{{ database.user }}"
   when: mysql_backup.credentials_handling == 'static'
+
+### Start of legacy handling. Can remove in a few months.
+- name: Catch legacy static password handling.
+  ansible.builtin.stat:
+    path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}"
+  register: _legacy_static_creds
+  delegate_to: localhost
+  run_once: true
+  when: mysql_backup.credentials_handling == 'static'
+
+- name: Create new static password location if it doesn't exist.
+  ansible.builtin.file:
+    path: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: 0755
+  delegate_to: localhost
+  run_once: true
+  when: mysql_backup.credentials_handling == 'static'
+
+- name: Copy legacy static password to new location.
+  ansible.builtin.copy:
+    src: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}"
+    dest: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}"
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: 0644
+  delegate_to: localhost
+  run_once: true
+  when:
+    - mysql_backup.credentials_handling == 'static'
+    - _legacy_static_creds is defined
+    - _legacy_static_creds.stat.exists
+
+- name: Delete legacy static password file if it exists.
+  ansible.builtin.file:
+    path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}"
+    state: absent
+  delegate_to: localhost
+  run_once: true
+  when:
+    - mysql_backup.credentials_handling == 'static'
+    - _legacy_static_creds is defined
+    - _legacy_static_creds.stat.exists
+### End of legacy handling.
+
 - set_fact:
-    _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}"
+    _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}"
   when: mysql_backup.credentials_handling == 'static'
 # Rotate: create user/pwd on each build.
 - set_fact:

From b6ea0206e8f83f7e2459babd0622403b955276af Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 31 May 2022 16:26:10 +0200
Subject: [PATCH 012/139] Making contents of deploy tar 'ownerless'. (#117)

---
 roles/deploy_code/tasks/cleanup.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 4e648d25..03530f22 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -22,7 +22,7 @@
 
 - name: Create a tarball of the deployed codebases.
   ansible.builtin.command:
-    cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}"
+    cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1

From 1c2b206d009b21d10b96c434168c15ac5700b96c Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Tue, 7 Jun 2022 15:48:04 +0100
Subject: [PATCH 013/139] Implement file syncing (#124)

* Add a files_sync role.

* Clear up some comments, add temp_dir variable and use rsync with command module instead of synchronize module, in files_sync role.

* Need trailing slash on src when syncing files to destination server so the contents get synced and not the parent directory itself.

* Use rsync instead of copy when syncing files to target server as copy is so slow.

* Ensure file sync tasks are run only once.
---
 roles/sync/files_sync/defaults/main.yml | 16 ++++++++++++++++
 roles/sync/files_sync/tasks/main.yml    |  7 +++++++
 roles/sync/files_sync/tasks/sync.yml    | 21 +++++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 roles/sync/files_sync/defaults/main.yml
 create mode 100644 roles/sync/files_sync/tasks/main.yml
 create mode 100644 roles/sync/files_sync/tasks/sync.yml

diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml
new file mode 100644
index 00000000..5b65b1fb
--- /dev/null
+++ b/roles/sync/files_sync/defaults/main.yml
@@ -0,0 +1,16 @@
+---
+files_sync:
+  directories:
+    - source:
+        # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH!
+        files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files"
+        # Host that contains source files.
+        host: "localhost"
+        # Location on deploy server where source files get copied to first. NO TRAILING SLASH.
+        temp_dir: "/tmp"
+        # Used to create directory in /tmp.
+        build_id: mybuildprod
+      target:
+        # Location of the files to sync to. DO NOT INCLUDE TRAILING SLASH!
+        files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files"
+        build_id: mybuilddev
diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml
new file mode 100644
index 00000000..47c3667a
--- /dev/null
+++ b/roles/sync/files_sync/tasks/main.yml
@@ -0,0 +1,7 @@
+---
+- name: Sync files.
+  include_tasks: "sync.yml"
+  with_items: "{{ files_sync.directories }}"
+  loop_control:
+    loop_var: files
+  run_once: true
diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml
new file mode 100644
index 00000000..e2160008
--- /dev/null
+++ b/roles/sync/files_sync/tasks/sync.yml
@@ -0,0 +1,21 @@
+---
+- name: Create a temporary directory for source files on localhost.
+  ansible.builtin.file:
+    path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+  delegate_to: localhost
+  run_once: true
+
+- name: Copy the source files onto the deploy server.
+  ansible.builtin.command:
+    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/"
+  delegate_to: "localhost"
+  run_once: true
+
+- name: Copy the source files from the deploy server onto the destination server.
+  ansible.builtin.command:
+    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/"
+  delegate_to: "localhost"
+  run_once: true

From 1cd0ed0797c1d26a933b4a3e06e0279e4af6a05a Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Wed, 8 Jun 2022 12:47:34 +0100
Subject: [PATCH 014/139] Create Drupal-specific sync roles (#128)

* Add a drupal_sync_tasks role to run Drupal specific tasks during syncs.

* Move sync database_apply files into tasks subdirectory.

* Woops, used bad role names in sync database update roles.

* Remove feature reverting from drupal7 database_apply sync role.
---
 .../cache_clear-drupal7/tasks/main.yml        | 11 ++++++++
 .../cache_clear-drupal8/tasks/main.yml        | 11 ++++++++
 .../database_apply-drupal7/tasks/main.yml     | 12 +++++++++
 .../database_apply-drupal8/tasks/main.yml     | 26 +++++++++++++++++++
 .../database_apply/tasks/main.yml             |  6 +++++
 5 files changed, 66 insertions(+)
 create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
 create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
 create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
 create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
 create mode 100644 roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml

diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
new file mode 100644
index 00000000..ab9342ba
--- /dev/null
+++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
@@ -0,0 +1,11 @@
+---
+- name: Clear Drupal 7 cache.
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  run_once: true
diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
new file mode 100644
index 00000000..96c721ee
--- /dev/null
+++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -0,0 +1,11 @@
+---
+- name: Clear Drupal cache.
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  run_once: true
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
new file mode 100644
index 00000000..f520514e
--- /dev/null
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
@@ -0,0 +1,12 @@
+---
+- name: Apply Drupal database updates.
+  ansible.builtin.shell:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+
+- name: Clear the cache.
+  ansible.builtin.include_role:
+    name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
new file mode 100644
index 00000000..bb9cc73c
--- /dev/null
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
@@ -0,0 +1,26 @@
+---
+- name: Apply Drupal database updates.
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+
+- name: Import configuration.
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  when:
+    - site.config_import_command
+
+- name: Clear the cache.
+  ansible.builtin.include_role:
+    name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml
new file mode 100644
index 00000000..58f6215a
--- /dev/null
+++ b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml
@@ -0,0 +1,6 @@
+---
+- name: Apply database updates.
+  ansible.builtin.include_role:
+    name: "sync/drupal_sync_tasks/database_apply/database_apply-{{ project_type }}"
+  when: deploy_operation == 'deploy'
+  run_once: true

From 904033f8bf3e1973d7b78c8ea2072426f1d0decb Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 10 Jun 2022 12:57:40 +0200
Subject: [PATCH 015/139] Fixing GRANT query for MySQL > 8.0. (#131)

* Fixing GRANT query for MySQL > 8.0.

* Create what though?
---
 roles/_init/defaults/main.yml                                | 1 +
 roles/database_backup/database_backup-mysql/tasks/deploy.yml | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index c4d15b78..790e9409 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -1,6 +1,7 @@
 ---
 # Common defaults. Given the "_init" role is mandatory,
 # this will ensure defaults to other roles too.
+# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index dbb39de7..b0328958 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -88,8 +88,9 @@
 # to allow this role to be looped over,
 # for multisites or projects with multiple databases.
 # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL.
+# As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
 - name: Create/update mysql user.
-  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}';"
+  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 

From 94715d8bd5363c640649e09bffe0e3b1574ceb90 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Mon, 13 Jun 2022 10:34:12 +0100
Subject: [PATCH 016/139] Use IF NOT EXISTS when creating database user as that
 command fails if static database users are being used. (#133)

---
 roles/database_backup/database_backup-mysql/tasks/deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index b0328958..f5a7130a 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -90,7 +90,7 @@
 # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL.
 # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
 - name: Create/update mysql user.
-  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
+  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 

From 42f12c81d0974648857d203139a154c736ac9731 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Thu, 16 Jun 2022 13:39:30 +0100
Subject: [PATCH 017/139] Attempt to fix syncs whenever the 'dump' type is used
 for source or target. (#149)

---
 .../database_sync-mysql/tasks/sync.yml          | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index c3ff0b9e..98eca0f3 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -38,12 +38,6 @@
 - name: Register remote dump name (from database).
   set_fact:
     mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2"
-  when: not database.source.type == 'dump'
-
-- name: Register existing remote dump name.
-  set_fact:
-    mysql_sync_source_dump_path: "{{ database.source.database }}"
-  when: database.source.type == 'dump'
 
 - name: Get source last known good build number.
   command:
@@ -70,7 +64,6 @@
   shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
   when:
-    - not database.source.type == 'dump'
     - database.source.fresh_db
 
 - name: Find source database host.
@@ -97,12 +90,6 @@
 - name: Register tmp target dump name.
   set_fact:
     mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2"
-  when: not database.target.type == 'dump'
-
-- name: Register final target dump name.
-  set_fact:
-    mysql_sync_target_dump_path: "{{ database.target.database }}"
-  when: database.target.type == 'dump'
 
 - name: Get target last known good build number.
   command:
@@ -139,21 +126,17 @@
 
 - name: Drop target database.
   shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'"
-  when: not database.target.type == 'dump'
 
 - name: Recreate target database.
   shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
-  when: not database.target.type == 'dump'
 
 - name: Repopulate database from dump.
   shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
-  when: not database.target.type == 'dump'
 
 - name: Remove tmp dump file.
   file:
     path: "{{ mysql_sync_target_dump_path }}"
     state: absent
-  when: not database.target.type == 'dump'
 
 - name: Enable all autoscale processes on source ASG.
   ansible.builtin.command: >

From a9c6abfb5c686d030a0671d8a20805b7d3b1be02 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 20 Jun 2022 11:31:01 +0200
Subject: [PATCH 018/139] Squashfs pr 1.x (#150)

* Adding SquashFS option to syncing.

* Make 'tarball' the default mount type so nothing existing breaks.

* Altering deploy_path so we can build in another location for SquashFS builds.

* Ensuring the build dir exists if doing a SquashFS build.

* Force symlink creation as the deploy dir may not exist yet.

* Running mount commands with sudo.

* Tweaking link command and destination for live links.

* Slight bug in link path handling for SquashFS.

* Checking for existing mount and using remount operation.

* Stop ce-deploy trying to delete from read-only SquashFS mount.

* Formatting error, these special jinja2 things are not filters - no spaces.

* Reloading services to make sure mounting doesn't fail.

* Picking more sensible loop var name.

* Moving to shell for mount check and fixing jinja2 filter names.

* Trying with the posix mount module instead of command.

* Working through user/sudo issues.

* Altering symlink handling slightly so we have the deploy directory always set.

* Simplifying 'when' checks.
---
 roles/_init/tasks/main.yml          |  51 +++++++++----
 roles/deploy_code/defaults/main.yml |  13 +++-
 roles/deploy_code/tasks/cleanup.yml | 108 +++++++++++++++++++++++++++-
 roles/live_symlink/tasks/main.yml   |  23 ++++--
 4 files changed, 171 insertions(+), 24 deletions(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index a1b6e204..53be27d3 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -1,27 +1,45 @@
 ---
 # Ensure default values for common variables.
 - name: Define deploy user.
-  set_fact:
+  ansible.builtin.set_fact:
     deploy_user: "{{ deploy_user | default('deploy') }}"
 - name: Define deploy base path.
-  set_fact:
+  ansible.builtin.set_fact:
     deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}"
 - name: Define mounted directory for assets.
-  set_fact:
+  ansible.builtin.set_fact:
     deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}"
 - name: Define webroot.
-  set_fact:
+  ansible.builtin.set_fact:
     webroot: "{{ webroot | default('web') }}"
 - name: Define build deploy path prefix.
-  set_fact:
+  ansible.builtin.set_fact:
     deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_"
 - name: Define build deploy path.
-  set_fact:
+  ansible.builtin.set_fact:
     deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}"
 - name: Define live_symlink dest.
-  set_fact:
+  ansible.builtin.set_fact:
     live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
 
+# Manipulate variables for SquashFS builds.
+- name: Define image builds base path.
+  ansible.builtin.set_fact:
+    build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}"
+  when: deploy_code.mount_type == "squashfs"
+- name: Define image builds build path prefix.
+  ansible.builtin.set_fact:
+    build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_"
+  when: deploy_code.mount_type == "squashfs"
+- name: Define live_symlink dest for image builds.
+  ansible.builtin.set_fact:
+    live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
+  when: deploy_code.mount_type == "squashfs"
+- name: Overwrite deploy and live_symlink paths if SquashFS deploy.
+  ansible.builtin.set_fact:
+    deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
+  when: deploy_code.mount_type == "squashfs"
+
 # Gather last known good build directly from symlink.
 # This can happen:
 # - when the first builds failed,
@@ -42,7 +60,7 @@
 
 # This is passed from caller.
 - name: Gather last known good build number.
-  set_fact:
+  ansible.builtin.set_fact:
     previous_build_number: "{{ previous_known_build_number }}"
 
 # - set_fact:
@@ -53,27 +71,34 @@
 
 # Make sure the deploy target exists.
 - name: Ensure deploy target directory exists.
-  file:
+  ansible.builtin.file:
     path: "{{ deploy_base_path }}"
     state: directory
 
+# Make sure the build target exists.
+- name: Ensure build target directory exists.
+  ansible.builtin.file:
+    path: "{{ build_base_path }}"
+    state: directory
+  when: deploy_code.mount_type == "squashfs"
+
 # Check for project specific init tasks.
 - name: Check that {{ project_type }}.yml exists.
-  stat:
+  ansible.builtin.stat:
     path: "{{ role_path }}/tasks/{{ project_type }}.yml"
   register: _project_type_task_result
   delegate_to: "localhost"
 
 # Project specific init tasks.
 - name: Include project init tasks.
-  include_tasks: "{{ project_type }}.yml"
+  ansible.builtin.include_tasks: "{{ project_type }}.yml"
   when:
     - _project_type_task_result.stat.exists
 
 - name: Define opcache cachetool path.
-  set_fact:
+  ansible.builtin.set_fact:
     cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}"
 
 - name: Ensure we have a cachetool binary.
-  import_role:
+  ansible.builtin.import_role:
     name: cli/cachetool
diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index d616bf08..f1f7cdb4 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -3,7 +3,7 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -15,12 +15,21 @@ deploy_code:
   templates: []
   # Number of builds to keep. Note this is independant of databases/dump.
   keep: 10
-  # Wether to sync the local deploy base to a shared destination, after successful build.
+  # Whether to sync the local deploy base to a shared destination, after successful build.
   mount_sync: ""
   # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy"
+  # Type of file to use for sync - 'squashfs' or 'tarball'
+  # @see the _init role for SquashFS build dir paths
+  # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour
+  mount_type: "tarball"
   # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash.
   perms_fix_path: ""
   # perms_fix_path: "www/sites/default"
+  # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount.
+  # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
+  services: []
+  # services:
+  #   - php8.0-fpm
   # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
   rebuild_infra: false
   # Details of API call to trigger. See api_call role.
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 03530f22..c722b049 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -6,19 +6,39 @@
   become: true
   when: "www_user != deploy_user"
 
-- name: Ensure permissions are set on directory.
+- name: Ensure permissions are set on deploy directory.
   ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
     - deploy_code.perms_fix_path is defined
     - deploy_code.perms_fix_path | length > 1
+    - deploy_code.mount_type != "squashfs"
 
-- name: Delete codebases.
+- name: Ensure permissions are set on builds directory.
+  ansible.builtin.shell:
+    cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
+  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  when:
+    - deploy_code.perms_fix_path is defined
+    - deploy_code.perms_fix_path | length > 1
+    - deploy_code.mount_type == "squashfs"
+
+- name: Delete codebases from deploy directory.
   ansible.builtin.file:
     name: "{{ deploy_path_prefix }}{{ item }}"
     state: absent
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  when:
+    - deploy_code.mount_type != "squashfs"
+
+- name: Delete codebases from builds directory.
+  ansible.builtin.file:
+    name: "{{ build_path_prefix }}{{ item }}"
+    state: absent
+  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  when:
+    - deploy_code.mount_type == "squashfs"
 
 - name: Create a tarball of the deployed codebases.
   ansible.builtin.command:
@@ -26,6 +46,26 @@
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "tarball"
+  run_once: true
+
+- name: Ensure older SquashFS images are deleted.
+  ansible.builtin.file:
+    path: "{{ build_base_path }}/deploy.sqsh"
+    state: absent
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+  run_once: true
+
+- name: Create a SquashFS image of the deployed codebases.
+  ansible.builtin.command:
+    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh"
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
   run_once: true
 
 - name: Create destination folder.
@@ -38,14 +78,76 @@
     - deploy_code.mount_sync | length > 1
   run_once: true
 
-- name: Move to final destination.
+- name: Move tar file to final destination.
   ansible.builtin.command:
     cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "tarball"
+  run_once: true
+
+- name: Move SquashFS image to final destination.
+  ansible.builtin.command:
+    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh"
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
   run_once: true
 
+- name: Copy SquashFS image to local server.
+  ansible.builtin.command:
+    cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh"
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+
+- name: Check if we have a mount already.
+  ansible.builtin.shell:
+    cmd: "mount | grep {{ deploy_base_path }}"
+  ignore_errors: true
+  register: _mount_check
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+
+- name: Unmount existing SquashFS image.
+  ansible.builtin.command:
+    cmd: "umount -l {{ deploy_base_path }}"
+  become: true
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _mount_check is succeeded
+
+- name: Reload any services that might be keeping the loop device busy.
+  ansible.builtin.service:
+    name: "{{ www_service }}"
+    state: reloaded
+  with_items: "{{ deploy_code.services }}"
+  loop_control:
+    loop_var: www_service
+  become: true
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _mount_check is succeeded
+    - deploy_code.services | length > 0
+
+- name: Mount new SquashFS image.
+  ansible.builtin.command:
+    cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop"
+  become: true
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+
 - name: Trigger an infrastructure rebuild.
   ansible.builtin.include_role:
     name: api_call
diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml
index 460ced9a..fa1d9e97 100644
--- a/roles/live_symlink/tasks/main.yml
+++ b/roles/live_symlink/tasks/main.yml
@@ -1,19 +1,28 @@
 ---
-- set_fact:
+- ansible.builtin.set_fact:
     _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ build_number }}"
 
-- set_fact:
+- ansible.builtin.set_fact:
     _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ previous_build_number }}"
   when: deploy_operation == 'revert'
 
+- ansible.builtin.set_fact:
+    _live_symlink_dest_target: "{{ live_symlink_dest }}"
+
+- ansible.builtin.set_fact:
+    _live_symlink_dest_target: "{{ live_symlink_build_dest }}"
+  when: deploy_code.mount_type == "squashfs"
+
 - name: Symlink build.
-  file:
+  ansible.builtin.file:
     src: "{{ _live_symlink_build_target }}"
-    dest: "{{ live_symlink_dest }}"
+    dest: "{{ _live_symlink_dest_target }}"
     state: link
+    follow: false
+    force: true
 
 - name: Generate additional templates.
-  template:
+  ansible.builtin.template:
     src: "{{ template.src }}"
     dest: "{{ deploy_path }}/{{ template.dest }}"
   with_items: "{{ live_symlink.templates }}"
@@ -24,10 +33,12 @@
     - deploy_operation == 'deploy'
 
 - name: Create additional symlinks.
-  file:
+  ansible.builtin.file:
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
+    follow: false
+    force: true
   with_items: "{{ live_symlink.symlinks }}"
   loop_control:
     loop_var: link

From a4de7710d2d7609aa9213c1f73951eaf27b23cbd Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Fri, 24 Jun 2022 13:00:24 +0100
Subject: [PATCH 019/139] Check deploy_code.mount_type is defined when setting
 facts in init role. (#155)

---
 roles/_init/tasks/main.yml | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 53be27d3..effa2790 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -26,19 +26,27 @@
 - name: Define image builds base path.
   ansible.builtin.set_fact:
     build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}"
-  when: deploy_code.mount_type == "squashfs"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 - name: Define image builds build path prefix.
   ansible.builtin.set_fact:
     build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_"
-  when: deploy_code.mount_type == "squashfs"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 - name: Define live_symlink dest for image builds.
   ansible.builtin.set_fact:
     live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
-  when: deploy_code.mount_type == "squashfs"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 - name: Overwrite deploy and live_symlink paths if SquashFS deploy.
   ansible.builtin.set_fact:
     deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
-  when: deploy_code.mount_type == "squashfs"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 
 # Gather last known good build directly from symlink.
 # This can happen:
@@ -80,7 +88,9 @@
   ansible.builtin.file:
     path: "{{ build_base_path }}"
     state: directory
-  when: deploy_code.mount_type == "squashfs"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 
 # Check for project specific init tasks.
 - name: Check that {{ project_type }}.yml exists.

From fe2aaf10f5bfc9124f5c1ab1cb6bff6e8e5b635c Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Tue, 28 Jun 2022 11:08:07 +0100
Subject: [PATCH 020/139] Make config imports during syncs optional (#157)

* Use a different variable for config imports during a sync so they can be optional.

* Actually, no. Use a variable to determine if config should be imported during a sync.

* Fix some logic with config import during syncs and add a comment to explain when the task runs.
---
 .../database_apply/database_apply-drupal8/tasks/main.yml        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
index bb9cc73c..e050af3f 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
@@ -9,6 +9,7 @@
   loop_control:
     loop_var: site
 
+# This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run.
 - name: Import configuration.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
@@ -20,6 +21,7 @@
     loop_var: site
   when:
     - site.config_import_command
+    - site.sync_config_import is not defined or site.sync_config_import
 
 - name: Clear the cache.
   ansible.builtin.include_role:

From 451227d28461c0afcc0bb854277fda17796d1d9e Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 29 Jun 2022 17:40:45 +0200
Subject: [PATCH 021/139] Squashfs pr 1.x (#153)

* Adding SquashFS option to syncing.

* Make 'tarball' the default mount type so nothing existing breaks.

* Altering deploy_path so we can build in another location for SquashFS builds.

* Ensuring the build dir exists if doing a SquashFS build.

* Force symlink creation as the deploy dir may not exist yet.

* Running mount commands with sudo.

* Tweaking link command and destination for live links.

* Slight bug in link path handling for SquashFS.

* Checking for existing mount and using remount operation.

* Stop ce-deploy trying to delete from read-only SquashFS mount.

* Formatting error, these special jinja2 things are not filters - no spaces.

* Reloading services to make sure mounting doesn't fail.

* Picking more sensible loop var name.

* Moving to shell for mount check and fixing jinja2 filter names.

* Trying with the posix mount module instead of command.

* Working through user/sudo issues.

* Altering symlink handling slightly so we have the deploy directory always set.

* Simplifying 'when' checks.

* Adding a revert behaviour for SquashFS.

* Making some wording a little less ambiguous.

* Fixing bad image path and adding clauses to stat check.

* Trying to make remounting SquashFS images a bit safer.

* Adding when clauses to SquashFS image path check.

* Removing revert code, as it cannot work.
---
 roles/deploy_code/tasks/cleanup.yml | 64 +++++++++++++++++++----------
 1 file changed, 42 insertions(+), 22 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index c722b049..e1412749 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -49,16 +49,6 @@
     - deploy_code.mount_type == "tarball"
   run_once: true
 
-- name: Ensure older SquashFS images are deleted.
-  ansible.builtin.file:
-    path: "{{ build_base_path }}/deploy.sqsh"
-    state: absent
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-  run_once: true
-
 - name: Create a SquashFS image of the deployed codebases.
   ansible.builtin.command:
     cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh"
@@ -96,6 +86,36 @@
     - deploy_code.mount_type == "squashfs"
   run_once: true
 
+- name: Check if we have a SquashFS image already there.
+  ansible.builtin.stat:
+    path: "{{ build_base_path }}/deploy.sqsh"
+  register: _deploy_code_mount_image
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+
+- name: Copy previous SquashFS image in case of rollback.
+  ansible.builtin.copy:
+    remote_src: true
+    force: true
+    src: "{{ build_base_path }}/deploy.sqsh"
+    dest: "{{ build_base_path }}/deploy_previous.sqsh"
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _deploy_code_mount_image.stat.islnk is defined
+
+- name: Ensure mounted SquashFS image is deleted.
+  ansible.builtin.file:
+    path: "{{ build_base_path }}/deploy.sqsh"
+    state: absent
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+
 - name: Copy SquashFS image to local server.
   ansible.builtin.command:
     cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh"
@@ -108,21 +128,11 @@
   ansible.builtin.shell:
     cmd: "mount | grep {{ deploy_base_path }}"
   ignore_errors: true
-  register: _mount_check
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-
-- name: Unmount existing SquashFS image.
-  ansible.builtin.command:
-    cmd: "umount -l {{ deploy_base_path }}"
-  become: true
+  register: _deploy_code_mount_check
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _mount_check is succeeded
 
 - name: Reload any services that might be keeping the loop device busy.
   ansible.builtin.service:
@@ -136,9 +146,19 @@
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _mount_check is succeeded
+    - _deploy_code_mount_check is succeeded
     - deploy_code.services | length > 0
 
+- name: Unmount existing SquashFS image.
+  ansible.builtin.command:
+    cmd: "umount {{ deploy_base_path }}"
+  become: true
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _deploy_code_mount_check is succeeded
+
 - name: Mount new SquashFS image.
   ansible.builtin.command:
     cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop"

From c7aae71b8903846fccbbd325464509282cb6a302 Mon Sep 17 00:00:00 2001
From: EmlynK <emlyn.kinzett@codeenigma.com>
Date: Tue, 16 Aug 2022 12:18:58 +0100
Subject: [PATCH 022/139] Add cache clears to Drupal deployments, before DB
 updates and stuff (#159)

* Add cache clears to Drupal deployments, before database updates and config import are run.

* Move cache clears out of meta roles and only run in database_apply role on non-initial builds.

* Add cache clear stuff to drupal7 database_apply role.
---
 roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++
 roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 7c2201e2..8b655bda 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -19,6 +19,11 @@
     loop_var: site
   when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
 
+- name: Clear the cache.
+  ansible.builtin.include_role:
+    name: "cache_clear/cache_clear-{{ project_type }}"
+  when: previous_build_number > 0
+
 - name: Apply Drupal database updates.
   shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index dc138585..23cb0b3b 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -32,6 +32,11 @@
     loop_var: site
   when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
 
+- name: Clear the cache.
+  ansible.builtin.include_role:
+    name: "cache_clear/cache_clear-{{ project_type }}"
+  when: previous_build_number > 0
+
 - name: Apply Drupal database updates.
   command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"

From 15b8daae0b9736f2a80878415a02155dd2acc8e0 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 5 Sep 2022 18:05:10 +0200
Subject: [PATCH 023/139] Avoid leaving exponentially growing sqsh files in
 build locations! (#164)

* Avoid leaving exponentially growing sqsh files in build locations!

* Use mount point instead of /tmp for stowing previous sqsh file.
---
 roles/deploy_code/tasks/cleanup.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index e1412749..b206b7ab 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -51,7 +51,7 @@
 
 - name: Create a SquashFS image of the deployed codebases.
   ansible.builtin.command:
-    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh"
+    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
@@ -95,17 +95,18 @@
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
 
-- name: Copy previous SquashFS image in case of rollback.
+- name: Copy previous SquashFS image to shared mount point in case of rollback.
   ansible.builtin.copy:
     remote_src: true
     force: true
     src: "{{ build_base_path }}/deploy.sqsh"
-    dest: "{{ build_base_path }}/deploy_previous.sqsh"
+    dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
     - _deploy_code_mount_image.stat.islnk is defined
+  run_once: true
 
 - name: Ensure mounted SquashFS image is deleted.
   ansible.builtin.file:

From 5cab4ae00d301e183ab1d17cf53bf289391b4a6b Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 6 Oct 2022 16:28:30 +0200
Subject: [PATCH 024/139] Exclude sqsh file pr 1.x (#167)

* Avoid leaving exponentially growing sqsh files in build locations!

* Use mount point instead of /tmp for stowing previous sqsh file.

* Making sure image/tarball filenames are build specific in /tmp.
---
 roles/deploy_code/tasks/cleanup.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index b206b7ab..353854b3 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -42,7 +42,7 @@
 
 - name: Create a tarball of the deployed codebases.
   ansible.builtin.command:
-    cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}"
+    cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
@@ -51,7 +51,7 @@
 
 - name: Create a SquashFS image of the deployed codebases.
   ansible.builtin.command:
-    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh"
+    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
@@ -70,7 +70,7 @@
 
 - name: Move tar file to final destination.
   ansible.builtin.command:
-    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar"
+    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
@@ -79,7 +79,7 @@
 
 - name: Move SquashFS image to final destination.
   ansible.builtin.command:
-    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh"
+    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh"
   when:
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1

From 459faba5e163ee6355d4bb56f9ad19042c7aa148 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 7 Oct 2022 16:05:35 +0200
Subject: [PATCH 025/139] Removing unnecessary lines in Drupal config
 generation. (#169)

* Removing unnecessary lines in Drupal config generation.

* Drupal 7 lacked the install_command var.
---
 .../tasks/settings.yml                        | 16 ++++++---------
 .../tasks/settings.yml                        | 20 ++++++++-----------
 .../database_apply-drupal7/tasks/main.yml     |  4 ++--
 3 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
index 277899a0..f8e3b55e 100644
--- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
@@ -1,25 +1,21 @@
 ---
-- name: set public files relative path.
-  set_fact:
+- name: Set public files relative path.
+  ansible.builtin.set_fact:
     build_public_file_path: "{{ site.public_files }}"
 
-- name: set public files path.
-  set_fact:
-    build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files"
-
-- name: set private files path.
-  set_fact:
+- name: Set private files path.
+  ansible.builtin.set_fact:
     build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files"
 
 - name: Ensure file permissions on site folder.
-  file:
+  ansible.builtin.file:
     path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
     state: directory
     mode: 0775
   become: "{{ false if www_user == deploy_user else true }}"
 
 - name: Generates settings.php file.
-  template:
+  ansible.builtin.template:
     src: "{{ item }}"
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php"
   with_first_found:
diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index 70d9b113..ebf182e8 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -1,29 +1,25 @@
 ---
-- name: set public files relative path.
-  set_fact:
+- name: Set public files relative path.
+  ansible.builtin.set_fact:
     build_public_file_path: "{{ site.public_files }}"
 
-- name: set public files path.
-  set_fact:
+- name: Set private files path.
+  ansible.builtin.set_fact:
     build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files"
 
-- name: set private files path.
-  set_fact:
-    build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files"
-
-- name: set config directory path.
-  set_fact:
+- name: Set config directory path.
+  ansible.builtin.set_fact:
     build_config_sync_directory: "{{ deploy_path }}/{{ site.config_sync_directory }}"
 
 - name: Ensure file permissions on site folder.
-  file:
+  ansible.builtin.file:
     path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
     state: directory
     mode: 0775
   become: "{{ false if www_user == deploy_user else true }}"
 
 - name: Generates settings.php file.
-  template:
+  ansible.builtin.template:
     src: "{{ item }}"
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php"
   with_first_found:
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 8b655bda..eb2edbfa 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -2,7 +2,7 @@
 
 - name: Install Drupal.
   shell:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y si"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -45,4 +45,4 @@
   include_tasks: ctools.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
-    loop_var: site
\ No newline at end of file
+    loop_var: site

From 200f59002c0459d63edc4d821f7b31ee94dde351 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 10 Oct 2022 14:31:48 +0200
Subject: [PATCH 026/139] Ensuring dump directory exists on backup step. (#172)

* Ensuring dump directory exists on backup step.

* Acts on shared storage, so can run once.
---
 .../database_backup-mysql/tasks/deploy-dump.yml  | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index d06b1543..80a6e5ef 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -1,11 +1,11 @@
 ---
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_database_name: "{{ database.database }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
 - name: Create initial database.
-  mysql_db:
+  community.mysql.mysql_db:
     name: "{{ _mysql_build_database_name }}"
     state: present
     login_host: "{{ _mysql_host }}"
@@ -14,7 +14,15 @@
   when: previous_build_number == 0
   run_once: true
 
+- name: Ensure the dump directory exists.
+  ansible.builtin.file:
+    path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+  run_once: true
+
 - name: Take a database dump.
-  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
   when: previous_build_number > 0
   run_once: true

From 2c7443277b94486bad5f3d7b57212c9426bf5e9f Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 14 Oct 2022 13:49:41 +0200
Subject: [PATCH 027/139] Allowing Drupal 7 jobs to disable cron. (#174)

---
 roles/cron/cron_drupal7/defaults/main.yml |  1 +
 roles/cron/cron_drupal7/tasks/cron.yml    |  2 +-
 roles/cron/cron_drupal7/tasks/job.yml     | 10 ++++++----
 roles/cron/cron_drupal7/tasks/main.yml    |  4 ++--
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml
index a09c1036..a6551ab6 100644
--- a/roles/cron/cron_drupal7/defaults/main.yml
+++ b/roles/cron/cron_drupal7/defaults/main.yml
@@ -8,6 +8,7 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
diff --git a/roles/cron/cron_drupal7/tasks/cron.yml b/roles/cron/cron_drupal7/tasks/cron.yml
index 2bc2c6c7..056d50e0 100644
--- a/roles/cron/cron_drupal7/tasks/cron.yml
+++ b/roles/cron/cron_drupal7/tasks/cron.yml
@@ -1,6 +1,6 @@
 ---
 - name: Setup Drupal 7 cron tasks.
-  include_tasks:
+  ansible.builtin.include_tasks:
     file: job.yml
   with_items: "{{ site.cron }}"
   loop_control:
diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml
index 841c4b51..0f1d3203 100644
--- a/roles/cron/cron_drupal7/tasks/job.yml
+++ b/roles/cron/cron_drupal7/tasks/job.yml
@@ -1,10 +1,10 @@
 ---
 - name: Define cron job command.
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}"
 
 - name: Define cron job command if deferred (ASG).
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\""
   when:
     - drupal.defer is defined
@@ -13,7 +13,7 @@
     - drupal.defer_target | length > 0
 
 - name: Define cron job command for differing deploy users.
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'"
   when:
     - www_user != deploy_user
@@ -21,9 +21,11 @@
     - drupal.defer
 
 - name: Setup Drupal cron tasks on app server.
-  cron:
+  ansible.builtin.cron:
     name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}"
     minute: "{{ entry.minute }}"
     hour: "{{ entry.hour | default(omit) }}"
     job: "{{ _cron_job_command }}"
+    state: present
+    disabled: "{{ entry.disabled | default(omit) }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml
index 75fa05c7..cf0f80f6 100644
--- a/roles/cron/cron_drupal7/tasks/main.yml
+++ b/roles/cron/cron_drupal7/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Setup PATH in crontab.
-  cron:
+  ansible.builtin.cron:
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
@@ -10,7 +10,7 @@
     - drupal.defer
 
 - name: Create site cron entries.
-  include_tasks:
+  ansible.builtin.include_tasks:
     file: cron.yml
   with_items: "{{ drupal.sites }}"
   loop_control:

From 03852117a42d9aa62e30d3161ae0b418d6e440bf Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 14 Oct 2022 17:41:52 +0200
Subject: [PATCH 028/139] Suppress db revert pr 1.x (#177)

* Supporting option to suppress reverting backups.

* Fixing namespacing in database_backup role.
---
 .../tasks/cleanup-dump.yml                    |  2 +-
 .../tasks/cleanup-rolling.yml                 |  2 +-
 .../database_backup-mysql/tasks/cleanup.yml   | 12 ++++----
 .../tasks/deploy-none.yml                     |  6 ++--
 .../tasks/deploy-rolling.yml                  |  8 +++---
 .../database_backup-mysql/tasks/deploy.yml    | 28 +++++++++----------
 .../database_backup-mysql/tasks/main.yml      |  2 +-
 .../tasks/revert-dump.yml                     |  2 +-
 .../database_backup-mysql/tasks/revert.yml    | 14 ++++++----
 roles/database_backup/defaults/main.yml       |  1 +
 roles/database_backup/tasks/main.yml          |  4 +--
 11 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
index d5774a78..a9ea9ae2 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
@@ -1,7 +1,7 @@
 ---
 # We assume it's safe to only go back 50 build back.
 - name: Delete mysql dumps.
-  file:
+  ansible.builtin.file:
     path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2"
     state: absent
   with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
index b131f8e6..2bdf29ee 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
@@ -1,7 +1,7 @@
 ---
 # We assume it's safe to only go back 50 build back.
 - name: Delete mysql databases.
-  mysql_db:
+  community.mysql.mysql_db:
     name: "{{ database.database }}_{{ item }}"
     state: absent
     login_host: "{{ _mysql_host }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
index 71570baa..ac71fcbb 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
@@ -1,23 +1,23 @@
 ---
 - name: Grab mysql credentials.
-  fetch:
+  ansible.builtin.fetch:
     src: "{{ database.credentials_file }}"
     dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini"
     flat: true
 
 # Credentials for "managing" databases.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
 
-- include_tasks: "cleanup-{{ mysql_backup.handling }}.yml"
+- ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml"
 
 # We assume it's safe to only go back 50 build back.
 - name: Delete mysql users.
-  mysql_user:
+  community.mysql.mysql_user:
     name: "{{ database.user }}_{{ item }}"
     host_all: true
     state: absent
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index 65c68978..dea8fe9f 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -1,11 +1,11 @@
 ---
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_database_name: "{{ database.database }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
 - name: Create initial database.
-  mysql_db:
+  community.mysql.mysql_db:
     name: "{{ _mysql_build_database_name }}"
     state: present
     login_host: "{{ _mysql_host }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 1db841b0..71fa7580 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -1,17 +1,17 @@
 ---
 # Setup database names.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_database_name: "{{ database.database }}_{{ build_number }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}"
   # Note: we don't use the mysql_db Ansible module on purpose.
   # If database already exists, we want to fail and not override it
   # with previous build.
 - name: Create new database.
-  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};"
   run_once: true
 
 - name: Populate new database.
-  shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
+  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index f5a7130a..fc06be74 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -1,27 +1,27 @@
 ---
 - name: Grab mysql credentials.
-  fetch:
+  ansible.builtin.fetch:
     src: "{{ database.credentials_file }}"
     dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini"
     flat: true
 
 # Credentials for "managing" databases.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
 # Credentials for the app itself.
 # Manual: reuse given creds. Note the user gets ignored.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_user_name: "{{ _mysql_user }}"
   when: mysql_backup.credentials_handling == 'manual'
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_password: "{{ _mysql_password }}"
   when: mysql_backup.credentials_handling == 'manual'
 # Static: create/update user/pwd only once.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_user_name: "{{ database.user }}"
   when: mysql_backup.credentials_handling == 'static'
 
@@ -71,18 +71,18 @@
     - _legacy_static_creds.stat.exists
 ### End of legacy handling.
 
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}"
   when: mysql_backup.credentials_handling == 'static'
 # Rotate: create user/pwd on each build.
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_user_name: "{{ database.user }}_{{ build_number }}"
   when: mysql_backup.credentials_handling == 'rotate'
-- set_fact:
+- ansible.builtin.set_fact:
     _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}"
   when: mysql_backup.credentials_handling == 'rotate'
 
-- include_tasks: "deploy-{{ mysql_backup.handling }}.yml"
+- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml"
 
 # We append privileges instead of replacing,
 # to allow this role to be looped over,
@@ -90,11 +90,11 @@
 # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL.
 # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
 - name: Create/update mysql user.
-  command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 
-- set_fact:
+- ansible.builtin.set_fact:
     _build_database:
       original: "{{ database }}"
       name: "{{ _mysql_build_database_name }}"
@@ -102,5 +102,5 @@
       password: "{{ _mysql_build_password }}"
       host: "{{ _mysql_host }}"
 
-- set_fact:
+- ansible.builtin.set_fact:
     build_databases: "{{ build_databases + [ _build_database ] }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/main.yml b/roles/database_backup/database_backup-mysql/tasks/main.yml
index ec10df75..b015ac09 100644
--- a/roles/database_backup/database_backup-mysql/tasks/main.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Take database backup.
-  include_tasks: "{{ deploy_operation }}.yml"
+  ansible.builtin.include_tasks: "{{ deploy_operation }}.yml"
   with_items: "{{ mysql_backup.databases }}"
   loop_control:
     loop_var: database
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 735a9805..f914ee78 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -1,5 +1,5 @@
 ---
 - name: Revert database from dump.
-  shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}"
+  ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}"
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml
index ba054b44..f6076262 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml
@@ -1,17 +1,19 @@
 ---
 - name: Grab mysql credentials.
-  fetch:
+  ansible.builtin.fetch:
       src: "{{ database.credentials_file }}"
       dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini"
       flat: true
 
 # Credentials for "managing" databases.
-- set_fact:
+- ansible.builtin.set_fact:
       _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
       _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
-- set_fact:
+- ansible.builtin.set_fact:
       _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
 
-- include_tasks: "revert-{{ mysql_backup.handling }}.yml"
-  when: previous_build_number > 0
+- ansible.builtin.include_tasks: "revert-{{ mysql_backup.handling }}.yml"
+  when:
+    - previous_build_number > 0
+    - database_backup.revert
diff --git a/roles/database_backup/defaults/main.yml b/roles/database_backup/defaults/main.yml
index 9201fa20..df4881c0 100644
--- a/roles/database_backup/defaults/main.yml
+++ b/roles/database_backup/defaults/main.yml
@@ -2,3 +2,4 @@
 database_backup:
   engines:
     - mysql
+  revert: true
\ No newline at end of file
diff --git a/roles/database_backup/tasks/main.yml b/roles/database_backup/tasks/main.yml
index 83adedf9..829536c2 100644
--- a/roles/database_backup/tasks/main.yml
+++ b/roles/database_backup/tasks/main.yml
@@ -1,9 +1,9 @@
 ---
-- set_fact:
+- ansible.builtin.set_fact:
     build_databases: []
 
 - name: Take database dumps.
-  include_role:
+  ansible.builtin.include_role:
     name: "database_backup/database_backup-{{ engine }}"
   with_items: "{{ database_backup.engines }}"
   loop_control:

From 86f138a15aafe6dd75b2eba1b04ceafb39351782 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 18 Nov 2022 15:05:04 +0100
Subject: [PATCH 029/139] Fixing bad assumption that databases will have TCP
 connections. (#179)

---
 .../database_backup/database_backup-mysql/tasks/deploy.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index fc06be74..5442230d 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -89,11 +89,16 @@
 # for multisites or projects with multiple databases.
 # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL.
 # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
-- name: Create/update mysql user.
+- name: Create/update mysql user for TCP connections.
   ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 
+- name: Create/update mysql user for unix socket connections.
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';"
+  when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
+  run_once: true
+
 - ansible.builtin.set_fact:
     _build_database:
       original: "{{ database }}"

From b23654081e8f8a5058d97b2e8b17a785d9a548f2 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 9 Dec 2022 16:05:49 +0100
Subject: [PATCH 030/139] Handling the 'drush deploy' command more elegantly
 for Drupal 8+. (#180)

* Handling the 'drush deploy' command more elegantly for Drupal 8+.

* Changing logic to AND for cache rebuild.
---
 .../database_apply/database_apply-drupal8/tasks/main.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 23cb0b3b..3473d9e8 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -22,7 +22,7 @@
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
-  when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
+  when: (previous_build_number == 0) or (site.force_install is defined and site.force_install)
 
 - name: Fix permissions on Drupal directory.
   shell:
@@ -30,12 +30,14 @@
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
-  when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
+  when: (previous_build_number == 0) or (site.force_install is defined and site.force_install)
 
 - name: Clear the cache.
   ansible.builtin.include_role:
     name: "cache_clear/cache_clear-{{ project_type }}"
-  when: previous_build_number > 0
+  when:
+    - previous_build_number > 0
+    - site.config_import_command != 'deploy'
 
 - name: Apply Drupal database updates.
   command:
@@ -46,6 +48,7 @@
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
+  when: site.config_import_command != 'deploy'
 
 - name: Import configuration.
   command:

From 58069b60c45aef39a655e2d0976b123af5a82fbf Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 9 Dec 2022 16:08:26 +0100
Subject: [PATCH 031/139] Attempt to clear the opcache during Drupal
 deployments. (#182)

Co-authored-by: Emlyn Kinzett <emlyn.kinzett@codeenigma.com>
---
 roles/_meta/deploy-drupal7/tasks/main.yml                  | 2 ++
 roles/_meta/deploy-drupal8/tasks/main.yml                  | 2 ++
 roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++
 roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++
 4 files changed, 14 insertions(+)

diff --git a/roles/_meta/deploy-drupal7/tasks/main.yml b/roles/_meta/deploy-drupal7/tasks/main.yml
index 93e6b1b7..cecf3ea0 100644
--- a/roles/_meta/deploy-drupal7/tasks/main.yml
+++ b/roles/_meta/deploy-drupal7/tasks/main.yml
@@ -24,6 +24,8 @@
     name: sanitize/admin_creds
 - import_role:
     name: live_symlink
+- import_role:
+    name: cache_clear/cache_clear-opcache
 - import_role:
     name: cache_clear/cache_clear-drupal7
 - import_role:
diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml
index 50804566..6ce31d4f 100644
--- a/roles/_meta/deploy-drupal8/tasks/main.yml
+++ b/roles/_meta/deploy-drupal8/tasks/main.yml
@@ -26,6 +26,8 @@
     name: sanitize/admin_creds
 - import_role:
     name: live_symlink
+- import_role:
+    name: cache_clear/cache_clear-opcache
 - import_role:
     name: cache_clear/cache_clear-drupal8
 - import_role:
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index eb2edbfa..fe25bcaa 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -24,6 +24,11 @@
     name: "cache_clear/cache_clear-{{ project_type }}"
   when: previous_build_number > 0
 
+- name: Clear the opcache.
+  ansible.builtin.include_role:
+    name: cache_clear/cache_clear-opcache
+  when: previous_build_number > 0
+
 - name: Apply Drupal database updates.
   shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 3473d9e8..6dd13e22 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -39,6 +39,11 @@
     - previous_build_number > 0
     - site.config_import_command != 'deploy'
 
+- name: Clear the opcache.
+  ansible.builtin.include_role:
+    name: cache_clear/cache_clear-opcache
+  when: previous_build_number > 0
+
 - name: Apply Drupal database updates.
   command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"

From 3ea0bfe4324ea6b9e7acadebba771b4264a764e4 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 9 Dec 2022 18:32:18 +0100
Subject: [PATCH 032/139] Better drush deploy support pr 1.x (#185)

* Handling the 'drush deploy' command more elegantly for Drupal 8+.

* Changing logic to AND for cache rebuild.

* Em's clear cache code needs to run for each 'site' too.
---
 roles/_init/defaults/main.yml                              | 2 +-
 roles/database_apply/database_apply-drupal8/tasks/main.yml | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 790e9409..432485d9 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -10,7 +10,7 @@ drupal:
       public_files: "sites/default/files"
       # Drupal 8 variables
       config_sync_directory: "config/sync"
-      config_import_command: "" # i.e. "cim"
+      config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 6dd13e22..16f0417b 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -35,6 +35,9 @@
 - name: Clear the cache.
   ansible.builtin.include_role:
     name: "cache_clear/cache_clear-{{ project_type }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
   when:
     - previous_build_number > 0
     - site.config_import_command != 'deploy'

From b83534407d373150382dfd431802f5fbf31dc2d4 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 30 Dec 2022 13:22:01 +0200
Subject: [PATCH 033/139] Cron job schedule params pr 1.x (#190)

* cron job schedule params, namespaces

* drupal7-cron-params

* cron-job-schedule-params-fix-defaults
---
 roles/cron/cron_drupal7/defaults/main.yml |  3 +++
 roles/cron/cron_drupal7/tasks/job.yml     |  3 +++
 roles/cron/cron_drupal8/defaults/main.yml |  4 ++++
 roles/cron/cron_drupal8/tasks/cron.yml    |  2 +-
 roles/cron/cron_drupal8/tasks/job.yml     | 13 +++++++++----
 roles/cron/cron_drupal8/tasks/main.yml    |  4 ++--
 6 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml
index a6551ab6..ef3ab79f 100644
--- a/roles/cron/cron_drupal7/defaults/main.yml
+++ b/roles/cron/cron_drupal7/defaults/main.yml
@@ -7,6 +7,9 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
           # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml
index 0f1d3203..dd18979c 100644
--- a/roles/cron/cron_drupal7/tasks/job.yml
+++ b/roles/cron/cron_drupal7/tasks/job.yml
@@ -25,6 +25,9 @@
     name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}"
     minute: "{{ entry.minute }}"
     hour: "{{ entry.hour | default(omit) }}"
+    day: "{{ entry.day | default(omit) }}"
+    weekday: "{{ entry.weekday | default(omit) }}"
+    month: "{{ entry.month | default(omit) }}"
     job: "{{ _cron_job_command }}"
     state: present
     disabled: "{{ entry.disabled | default(omit) }}"
diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml
index 61e2ef8f..913eb530 100644
--- a/roles/cron/cron_drupal8/defaults/main.yml
+++ b/roles/cron/cron_drupal8/defaults/main.yml
@@ -7,7 +7,11 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
diff --git a/roles/cron/cron_drupal8/tasks/cron.yml b/roles/cron/cron_drupal8/tasks/cron.yml
index 2f0c6116..1ed40cc3 100644
--- a/roles/cron/cron_drupal8/tasks/cron.yml
+++ b/roles/cron/cron_drupal8/tasks/cron.yml
@@ -1,6 +1,6 @@
 ---
 - name: Setup Drupal cron tasks.
-  include_tasks:
+  ansible.builtin.include_tasks:
     file: job.yml
   with_items: "{{ site.cron }}"
   loop_control:
diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml
index bcdac2af..908abff6 100644
--- a/roles/cron/cron_drupal8/tasks/job.yml
+++ b/roles/cron/cron_drupal8/tasks/job.yml
@@ -1,10 +1,10 @@
 ---
 - name: Define cron job command.
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}"
 
 - name: Define cron job command if deferred (ASG).
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\""
   when:
     - drupal.defer is defined
@@ -13,7 +13,7 @@
     - drupal.defer_target | length > 0
 
 - name: Define cron job command for differing deploy users.
-  set_fact:
+  ansible.builtin.set_fact:
     _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'"
   when:
     - www_user != deploy_user
@@ -21,9 +21,14 @@
     - drupal.defer
 
 - name: Setup Drupal cron tasks on app server.
-  cron:
+  ansible.builtin.cron:
     name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}"
     minute: "{{ entry.minute }}"
     hour: "{{ entry.hour | default(omit) }}"
+    day: "{{ entry.day | default(omit) }}"
+    weekday: "{{ entry.weekday | default(omit) }}"
+    month: "{{ entry.month | default(omit) }}"
     job: "{{ _cron_job_command }}"
+    state: present
+    disabled: "{{ entry.disabled | default(omit) }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml
index 75fa05c7..cf0f80f6 100644
--- a/roles/cron/cron_drupal8/tasks/main.yml
+++ b/roles/cron/cron_drupal8/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Setup PATH in crontab.
-  cron:
+  ansible.builtin.cron:
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
@@ -10,7 +10,7 @@
     - drupal.defer
 
 - name: Create site cron entries.
-  include_tasks:
+  ansible.builtin.include_tasks:
     file: cron.yml
   with_items: "{{ drupal.sites }}"
   loop_control:

From e97e852235ff29185e7d52ceffe9c28f8e87a11f Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 23 Jan 2023 17:21:16 +0100
Subject: [PATCH 034/139] Adding option to stop services that might interfere
 with a squashfs mount. (#193)

---
 roles/deploy_code/defaults/main.yml |  5 ++++-
 roles/deploy_code/tasks/cleanup.yml | 33 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index f1f7cdb4..f9b84739 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -25,11 +25,14 @@ deploy_code:
   # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash.
   perms_fix_path: ""
   # perms_fix_path: "www/sites/default"
-  # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount.
+  # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount.
   # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
   services: []
   # services:
   #   - php8.0-fpm
+  # What action to take against the services, 'reload' or 'stop'.
+  # Busy websites will require a hard stop of services to achieve the umount command.
+  service_action: reload
   # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
   rebuild_infra: false
   # Details of API call to trigger. See api_call role.
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 353854b3..a5da0d87 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -144,6 +144,23 @@
     loop_var: www_service
   become: true
   when:
+    - deploy_code.service_action == "reload"
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _deploy_code_mount_check is succeeded
+    - deploy_code.services | length > 0
+
+- name: Stop any services that might be keeping the loop device busy.
+  ansible.builtin.service:
+    name: "{{ www_service }}"
+    state: stopped
+  with_items: "{{ deploy_code.services }}"
+  loop_control:
+    loop_var: www_service
+  become: true
+  when:
+    - deploy_code.service_action == "stop"
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
@@ -169,6 +186,22 @@
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
 
+- name: Start any services we stopped.
+  ansible.builtin.service:
+    name: "{{ www_service }}"
+    state: started
+  with_items: "{{ deploy_code.services }}"
+  loop_control:
+    loop_var: www_service
+  become: true
+  when:
+    - deploy_code.service_action == "stop"
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+    - _deploy_code_mount_check is succeeded
+    - deploy_code.services | length > 0
+
 - name: Trigger an infrastructure rebuild.
   ansible.builtin.include_role:
     name: api_call

From 7bbb77fbd07bedcbc9d19f0d5354b03b1b34186b Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 25 Jan 2023 15:32:19 +0100
Subject: [PATCH 035/139] Drush refactor pr 1.x (#197)

* Updating docs.

* Consistent drush handling in cron.

* Linting cruft and removing unimplemented drush.yml handling from D7.

* Refactor drush role to support new GitHub release format.

* Updating drush README.

* Casting the first part of the drush version string as an integer.

* Refactoring integer handling.

* Major version string will always be a string, assuming below 8 will never exist.

* Downloaded drush archive is remote, not local.

* Refactoring handling of .bin directory.

* Supporting drush-launcher.

* Removing drush installation support for Drupal 8 and above.

* Bad variable name in Drupal cron jobs.

* Fixing maintenance mode command linting and consistency.

* Moving chdir to args to see if it helps.

* Trying an actual 'cd' command inline.

* Cannot '&&' in a cmd.

* Trying drush's --root flag to set Drupal path.

* With --root we do not need chdir for drush.

* Revert "With --root we do not need chdir for drush."

This reverts commit d07aaf798a8f5699ed669f9411db2810e2bcd9d8.
---
 docs/_Sidebar.md                              |  2 +
 docs/roles/_init.md                           |  6 ++-
 docs/roles/_meta/deploy-simplesamlphp.md      |  9 +++++
 docs/roles/api_call.md                        | 26 +++++++++++++
 docs/roles/cli/cachetool.md                   |  2 +-
 docs/roles/cli/drush.md                       | 10 +++--
 .../cron_database_backup-mysql.md             |  1 +
 docs/roles/cron/cron_drupal7.md               |  6 ++-
 docs/roles/cron/cron_drupal8.md               |  7 +++-
 docs/roles/database_backup.md                 |  2 +-
 .../database_backup/database_backup-mysql.md  |  1 +
 docs/roles/deploy_code.md                     | 37 ++++++++++++++++++-
 .../sync/database_sync/database_sync-mysql.md | 10 +++++
 roles/_init/README.md                         |  6 ++-
 roles/_init/defaults/main.yml                 |  1 +
 roles/_init/tasks/drupal7.yml                 |  2 +-
 roles/_init/tasks/drupal8.yml                 |  6 +--
 roles/api_call/README.md                      | 17 +++++++++
 roles/cli/cachetool/README.md                 |  2 +-
 roles/cli/drush/README.md                     | 10 +++--
 roles/cli/drush/defaults/main.yml             |  5 +--
 roles/cli/drush/tasks/main.yml                | 11 +++---
 .../config_generate-drupal7/tasks/drush.yml   | 10 -----
 .../config_generate-drupal7/tasks/main.yml    | 10 +----
 .../config_generate-drupal8/tasks/drush.yml   |  3 +-
 .../config_generate-drupal8/tasks/main.yml    |  4 +-
 .../config_generate-matomo/tasks/main.yml     |  3 +-
 .../config_generate-mautic/tasks/main.yml     | 11 +++---
 roles/config_generate/tasks/main.yml          |  7 ++--
 .../cron_database_backup-mysql/README.md      |  1 +
 roles/cron/cron_drupal7/README.md             |  6 ++-
 roles/cron/cron_drupal7/defaults/main.yml     |  1 +
 roles/cron/cron_drupal7/tasks/job.yml         |  2 +-
 roles/cron/cron_drupal8/README.md             |  7 +++-
 roles/cron/cron_drupal8/defaults/main.yml     |  3 +-
 roles/cron/cron_drupal8/tasks/job.yml         |  2 +-
 roles/database_backup/README.md               |  2 +-
 .../database_backup-mysql/README.md           |  1 +
 roles/deploy_code/README.md                   | 37 ++++++++++++++++++-
 .../tasks/main.yml                            |  2 +-
 .../tasks/offline.yml                         | 11 +++---
 .../tasks/online.yml                          | 10 +++--
 .../database_sync-mysql/README.md             | 10 +++++
 43 files changed, 233 insertions(+), 89 deletions(-)
 create mode 100644 docs/roles/_meta/deploy-simplesamlphp.md
 create mode 100644 docs/roles/api_call.md
 delete mode 100644 roles/config_generate/config_generate-drupal7/tasks/drush.yml

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index efa27c64..16139d8f 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -3,6 +3,7 @@
   - [Install](/install)
   - [Usage](/scripts)
   - [Roles](roles)
+     - [API call](/roles/api_call)
      - [Config](/roles/cache_clear)
        - [Drupal 7](/roles/cache_clear/cache_clear-drupal7)
        - [Drupal 8](/roles/cache_clear/cache_clear-drupal8)
@@ -37,6 +38,7 @@
        - [Drupal 8](/roles/_meta/deploy-drupal8)
        - [Matomo](/roles/_meta/deploy-matomo)
        - [Mautic](/roles/_meta/deploy-mautic)
+       - [SimpleSAMLphp](/roles/_meta/deploy-simplesamlphp)
      - [NPM](/roles/npm)
      - [Sync roles](/roles/sync)
        - [Database sync](/roles/sync/database_sync)
diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index f733d1a3..1c7e835e 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 ---
 # Common defaults. Given the "_init" role is mandatory,
 # this will ensure defaults to other roles too.
+# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
+_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"
       public_files: "sites/default/files"
       # Drupal 8 variables
       config_sync_directory: "config/sync"
-      config_import_command: "" # i.e. "cim"
+      config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
@@ -32,7 +34,7 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-
+bin_directory: "/home/{{ deploy_user }}/.bin"
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/_meta/deploy-simplesamlphp.md b/docs/roles/_meta/deploy-simplesamlphp.md
new file mode 100644
index 00000000..3d82e671
--- /dev/null
+++ b/docs/roles/_meta/deploy-simplesamlphp.md
@@ -0,0 +1,9 @@
+# SimpleSAMLphp
+Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer.
+
+This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision:
+
+* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/docs/roles/api_call.md b/docs/roles/api_call.md
new file mode 100644
index 00000000..15ea2788
--- /dev/null
+++ b/docs/roles/api_call.md
@@ -0,0 +1,26 @@
+# API call
+
+Making RESTful API calls to other platforms.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+api_call:
+  type: gitlab
+  base_url: https://gitlab.example.com/api/v4/
+  path: projects # see documentation - https://docs.gitlab.com/ee/api/
+  method: GET
+  token: "" # empty means anonymous action
+  token_type: trigger # options are 'trigger' or 'personal'
+  variables: []
+  status_codes:
+    - 200
+    - 201
+    - 202
+```
+
+<!--ENDROLEVARS-->
diff --git a/docs/roles/cli/cachetool.md b/docs/roles/cli/cachetool.md
index 833eb8fc..9cba4a32 100644
--- a/docs/roles/cli/cachetool.md
+++ b/docs/roles/cli/cachetool.md
@@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user.
 ```yaml
 ---
 cachetool:
-  version: 8.2.2
+  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/cli/drush.md b/docs/roles/cli/drush.md
index 30f02549..41b673c2 100644
--- a/docs/roles/cli/drush.md
+++ b/docs/roles/cli/drush.md
@@ -1,13 +1,15 @@
 # Drush
-Installs the `drush` command-line tool for the deploy user.
+Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23).
+
+For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush`  documentation](https://www.drush.org/latest/install/).
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
 ---
 drush:
-  # Note: This is the "default" version,
-  # but projects should define theirs in composer.json.
-  version: 8.2.2
+  # Where possible always load drush in your Drupal website with composer.
+  version: 8.4.11
   use_vendor: false
 ```
 
diff --git a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md
index ee71e2bf..4d5bdab5 100644
--- a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md
+++ b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md
@@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases.
 cron_mysql_backup:
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
   keep: 10
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
 
 ```
 
diff --git a/docs/roles/cron/cron_drupal7.md b/docs/roles/cron/cron_drupal7.md
index 8222ddaf..14356d15 100644
--- a/docs/roles/cron/cron_drupal7.md
+++ b/docs/roles/cron/cron_drupal7.md
@@ -14,12 +14,16 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/cron/cron_drupal8.md b/docs/roles/cron/cron_drupal8.md
index ae25adc4..bafd49b0 100644
--- a/docs/roles/cron/cron_drupal8.md
+++ b/docs/roles/cron/cron_drupal8.md
@@ -14,13 +14,16 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-  # Drush location when installed with Composer
-  drush_location: "vendor/drush/drush/drush"
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
 
 ```
 
diff --git a/docs/roles/database_backup.md b/docs/roles/database_backup.md
index 947c8053..772efa70 100644
--- a/docs/roles/database_backup.md
+++ b/docs/roles/database_backup.md
@@ -7,7 +7,7 @@ Generate backups for each build.
 database_backup:
   engines:
     - mysql
-
+  revert: true
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index 5b0d7289..b52413bf 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -7,6 +7,7 @@ Generate MySQL backups for each build.
 mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index bad0040f..4e371f28 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -11,7 +11,7 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -23,12 +23,45 @@ deploy_code:
   templates: []
   # Number of builds to keep. Note this is independant of databases/dump.
   keep: 10
-  # Wether to sync the local deploy base to a shared destination, after successful build.
+  # Whether to sync the local deploy base to a shared destination, after successful build.
   mount_sync: ""
   # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy"
+  # Type of file to use for sync - 'squashfs' or 'tarball'
+  # @see the _init role for SquashFS build dir paths
+  # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour
+  mount_type: "tarball"
   # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash.
   perms_fix_path: ""
   # perms_fix_path: "www/sites/default"
+  # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount.
+  # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
+  services: []
+  # services:
+  #   - php8.0-fpm
+  # What action to take against the services, 'reload' or 'stop'.
+  # Busy websites will require a hard stop of services to achieve the umount command.
+  service_action: reload
+  # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
+  rebuild_infra: false
+  # Details of API call to trigger. See api_call role.
+  api_call:
+    type: gitlab
+    base_url: https://gitlab.example.com/api/v4/
+    path: projects/1/ref/main/trigger/pipeline
+    method: POST
+    token: asdf-1234
+    token_type: trigger
+    variables: []
+    # example build parameters
+    #  - "[ENV]=dev"
+    #  - "[PLAY]=myserver.yml"
+    #  - "[RESOURCE]=myserver-example-com"
+    #  - "[REGION]=eu-west-1"
+    #  - "[EXTRA_PARAMS]=--force"
+    status_codes:
+      - 200
+      - 201
+      - 202
 
 ```
 
diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index bcb9c6ad..cf7e5311 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -5,6 +5,7 @@ Sync MySQL databases between environments.
 ```yaml
 ---
 mysql_sync:
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   databases:
     - source:
         # Name of the database to take a dump from.
@@ -20,6 +21,13 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
+        # Whether or not use to create a fresh database backup or use a nightly one.
+        fresh_db: true
+        # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default.
+        # This var is only used when fresh_db is set to "false".
+        dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
+        # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
@@ -30,6 +38,8 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuilddev
+        # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
 
 ```
 
diff --git a/roles/_init/README.md b/roles/_init/README.md
index f733d1a3..1c7e835e 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 ---
 # Common defaults. Given the "_init" role is mandatory,
 # this will ensure defaults to other roles too.
+# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
+_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"
       public_files: "sites/default/files"
       # Drupal 8 variables
       config_sync_directory: "config/sync"
-      config_import_command: "" # i.e. "cim"
+      config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
@@ -32,7 +34,7 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-
+bin_directory: "/home/{{ deploy_user }}/.bin"
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 432485d9..b38c1e59 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -25,3 +25,4 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
+bin_directory: "/home/{{ deploy_user }}/.bin"
\ No newline at end of file
diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml
index 6487564c..133b3d73 100644
--- a/roles/_init/tasks/drupal7.yml
+++ b/roles/_init/tasks/drupal7.yml
@@ -1,7 +1,7 @@
 ---
 - name: Define Drush path.
   set_fact:
-    drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}"
+    drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}"
 
 - name: Ensure we have Drush binary.
   import_role:
diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml
index 6487564c..ebe63b82 100644
--- a/roles/_init/tasks/drupal8.yml
+++ b/roles/_init/tasks/drupal8.yml
@@ -1,8 +1,4 @@
 ---
 - name: Define Drush path.
   set_fact:
-    drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}"
-
-- name: Ensure we have Drush binary.
-  import_role:
-    name: cli/drush
+    drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}"
diff --git a/roles/api_call/README.md b/roles/api_call/README.md
index 83d1ab28..15ea2788 100644
--- a/roles/api_call/README.md
+++ b/roles/api_call/README.md
@@ -6,4 +6,21 @@ Making RESTful API calls to other platforms.
 <!--ENDTOC-->
 
 <!--ROLEVARS-->
+## Default variables
+```yaml
+---
+api_call:
+  type: gitlab
+  base_url: https://gitlab.example.com/api/v4/
+  path: projects # see documentation - https://docs.gitlab.com/ee/api/
+  method: GET
+  token: "" # empty means anonymous action
+  token_type: trigger # options are 'trigger' or 'personal'
+  variables: []
+  status_codes:
+    - 200
+    - 201
+    - 202
+```
+
 <!--ENDROLEVARS-->
diff --git a/roles/cli/cachetool/README.md b/roles/cli/cachetool/README.md
index 833eb8fc..9cba4a32 100644
--- a/roles/cli/cachetool/README.md
+++ b/roles/cli/cachetool/README.md
@@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user.
 ```yaml
 ---
 cachetool:
-  version: 8.2.2
+  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/cli/drush/README.md b/roles/cli/drush/README.md
index 30f02549..41b673c2 100644
--- a/roles/cli/drush/README.md
+++ b/roles/cli/drush/README.md
@@ -1,13 +1,15 @@
 # Drush
-Installs the `drush` command-line tool for the deploy user.
+Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23).
+
+For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush`  documentation](https://www.drush.org/latest/install/).
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
 ---
 drush:
-  # Note: This is the "default" version,
-  # but projects should define theirs in composer.json.
-  version: 8.2.2
+  # Where possible always load drush in your Drupal website with composer.
+  version: 8.4.11
   use_vendor: false
 ```
 
diff --git a/roles/cli/drush/defaults/main.yml b/roles/cli/drush/defaults/main.yml
index 68ff2c66..746fce44 100644
--- a/roles/cli/drush/defaults/main.yml
+++ b/roles/cli/drush/defaults/main.yml
@@ -1,6 +1,5 @@
 ---
 drush:
-  # Note: This is the "default" version,
-  # but projects should define theirs in composer.json.
-  version: 8.2.2
+  # Where possible always load drush in your Drupal website with composer.
+  version: 8.4.11
   use_vendor: false
\ No newline at end of file
diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml
index 4d27aa3e..5c73be51 100644
--- a/roles/cli/drush/tasks/main.yml
+++ b/roles/cli/drush/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Check if we already have a global Drush.
-  stat:
+  ansible.builtin.stat:
     path: "{{ drush_bin }}"
   register: drush_global
   when:
@@ -16,18 +16,19 @@
   register: drush_global_directory
 
 - name: Check if installed Drush version match.
-  shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'"
+  ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'"
   register: drush_global_version
   when:
     - deploy_operation == 'deploy'
     - drush_global.stat.exists
     - not drush.use_vendor
 
-- name: Download Drush installer.
-  get_url:
+- name: Download Drush installer for Drupal 7.
+  ansible.builtin.get_url:
     url: "https://github.com/drush-ops/drush/releases/download/{{ drush.version }}/drush.phar"
     dest: "{{ drush_bin }}"
     mode: 0755
+    force: true
   when:
     - deploy_operation == 'deploy'
     - not drush.use_vendor
@@ -49,7 +50,7 @@
     - deploy_operation == 'deploy'
 
 - name: Trigger Drush init tasks.
-  command: "{{ drush_bin }} init -y"
+  ansible.builtin.command: "{{ drush_bin }} init -y"
   when:
     - deploy_operation == 'deploy'
     - not drush.use_vendor
diff --git a/roles/config_generate/config_generate-drupal7/tasks/drush.yml b/roles/config_generate/config_generate-drupal7/tasks/drush.yml
deleted file mode 100644
index 9c85eb9d..00000000
--- a/roles/config_generate/config_generate-drupal7/tasks/drush.yml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-
-- name: Generates drush.yml file.
-  template:
-    src: '{{ item }}'
-    dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml"
-  with_first_found:
-    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml.j2"
-    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml"
-    - "drush.yml.j2"
\ No newline at end of file
diff --git a/roles/config_generate/config_generate-drupal7/tasks/main.yml b/roles/config_generate/config_generate-drupal7/tasks/main.yml
index b984fd58..23781ded 100644
--- a/roles/config_generate/config_generate-drupal7/tasks/main.yml
+++ b/roles/config_generate/config_generate-drupal7/tasks/main.yml
@@ -1,14 +1,6 @@
 ---
 - name: Generates settings file.
-  include_tasks: settings.yml
+  ansible.builtin.include_tasks: settings.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
-
-#- name: Generates Drush settings file.
-#  include_tasks: drush.yml
-#  when:
-#    - site.base_url is defined
-#  with_items: "{{ drupal.sites }}"
-#  loop_control:
-#    loop_var: site
diff --git a/roles/config_generate/config_generate-drupal8/tasks/drush.yml b/roles/config_generate/config_generate-drupal8/tasks/drush.yml
index 9c85eb9d..f4b7ae7a 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/drush.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/drush.yml
@@ -1,7 +1,6 @@
 ---
-
 - name: Generates drush.yml file.
-  template:
+  ansible.builtin.template:
     src: '{{ item }}'
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml"
   with_first_found:
diff --git a/roles/config_generate/config_generate-drupal8/tasks/main.yml b/roles/config_generate/config_generate-drupal8/tasks/main.yml
index cd3c19fd..3db5bf60 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/main.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/main.yml
@@ -1,12 +1,12 @@
 ---
 - name: Generates settings file.
-  include_tasks: settings.yml
+  ansible.builtin.include_tasks: settings.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
 
 - name: Generates Drush settings file.
-  include_tasks: drush.yml
+  ansible.builtin.include_tasks: drush.yml
   when:
     - site.base_url is defined
   with_items: "{{ drupal.sites }}"
diff --git a/roles/config_generate/config_generate-matomo/tasks/main.yml b/roles/config_generate/config_generate-matomo/tasks/main.yml
index 505ff3ce..e5482e65 100644
--- a/roles/config_generate/config_generate-matomo/tasks/main.yml
+++ b/roles/config_generate/config_generate-matomo/tasks/main.yml
@@ -1,7 +1,6 @@
 ---
-
 - name: Generates config file.
-  template:
+  ansible.builtin.template:
     src: '{{ item }}'
     dest: "{{ deploy_path }}/{{ webroot }}/config/config.ini.php"
   with_first_found:
diff --git a/roles/config_generate/config_generate-mautic/tasks/main.yml b/roles/config_generate/config_generate-mautic/tasks/main.yml
index d1da20c5..2a9d63d0 100644
--- a/roles/config_generate/config_generate-mautic/tasks/main.yml
+++ b/roles/config_generate/config_generate-mautic/tasks/main.yml
@@ -1,15 +1,14 @@
 ---
-
-- name: set public files path.
-  set_fact:
+- name: Set public files path.
+  ansible.builtin.set_fact:
     build_image_path: "{{ mautic.image_path }}"
 
-- name: set private files path.
-  set_fact:
+- name: Set private files path.
+  ansible.builtin.set_fact:
     build_upload_dir: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_private_files"
 
 - name: Generates local.php file.
-  template:
+  ansible.builtin.template:
     src: '{{ item }}'
     dest: "{{ deploy_path }}/{{ webroot }}/app/config/local.php"
   with_first_found:
diff --git a/roles/config_generate/tasks/main.yml b/roles/config_generate/tasks/main.yml
index 3bce314b..2af08e87 100644
--- a/roles/config_generate/tasks/main.yml
+++ b/roles/config_generate/tasks/main.yml
@@ -1,12 +1,11 @@
 ---
-
 - name: Generate project specific configuration.
-  include_role:
+  ansible.builtin.include_role:
     name: "config_generate/config_generate-{{ project_type }}"
   when: deploy_operation == 'deploy'
 
 - name: Generate additional templates.
-  template:
+  ansible.builtin.template:
     src: "{{ template.src }}"
     dest: "{{ deploy_path }}/{{ template.dest }}"
   with_items: "{{ config_generate.templates }}"
@@ -17,7 +16,7 @@
     - deploy_operation == 'deploy'
 
 - name: Create additional symlinks.
-  file:
+  ansible.builtin.file:
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md
index ee71e2bf..4d5bdab5 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md
@@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases.
 cron_mysql_backup:
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
   keep: 10
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
 
 ```
 
diff --git a/roles/cron/cron_drupal7/README.md b/roles/cron/cron_drupal7/README.md
index 8222ddaf..14356d15 100644
--- a/roles/cron/cron_drupal7/README.md
+++ b/roles/cron/cron_drupal7/README.md
@@ -14,12 +14,16 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml
index ef3ab79f..419e9c4b 100644
--- a/roles/cron/cron_drupal7/defaults/main.yml
+++ b/roles/cron/cron_drupal7/defaults/main.yml
@@ -16,3 +16,4 @@ drupal:
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
\ No newline at end of file
diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml
index dd18979c..b89a0b05 100644
--- a/roles/cron/cron_drupal7/tasks/job.yml
+++ b/roles/cron/cron_drupal7/tasks/job.yml
@@ -1,7 +1,7 @@
 ---
 - name: Define cron job command.
   ansible.builtin.set_fact:
-    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}"
+    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
 
 - name: Define cron job command if deferred (ASG).
   ansible.builtin.set_fact:
diff --git a/roles/cron/cron_drupal8/README.md b/roles/cron/cron_drupal8/README.md
index ae25adc4..bafd49b0 100644
--- a/roles/cron/cron_drupal8/README.md
+++ b/roles/cron/cron_drupal8/README.md
@@ -14,13 +14,16 @@ drupal:
       cron: # These are the relevant parts for cron.
         - minute: "*/{{ 10 | random(start=1) }}"
           # hour: 2
+          # day:
+          # weekday:
+          # month:
           job: cron
+          # disabled: true
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-  # Drush location when installed with Composer
-  drush_location: "vendor/drush/drush/drush"
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
 
 ```
 
diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml
index 913eb530..9d4b90f6 100644
--- a/roles/cron/cron_drupal8/defaults/main.yml
+++ b/roles/cron/cron_drupal8/defaults/main.yml
@@ -16,5 +16,4 @@ drupal:
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-  # Drush location when installed with Composer
-  drush_location: "vendor/drush/drush/drush"
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml
index 908abff6..b89a0b05 100644
--- a/roles/cron/cron_drupal8/tasks/job.yml
+++ b/roles/cron/cron_drupal8/tasks/job.yml
@@ -1,7 +1,7 @@
 ---
 - name: Define cron job command.
   ansible.builtin.set_fact:
-    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}"
+    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
 
 - name: Define cron job command if deferred (ASG).
   ansible.builtin.set_fact:
diff --git a/roles/database_backup/README.md b/roles/database_backup/README.md
index 947c8053..772efa70 100644
--- a/roles/database_backup/README.md
+++ b/roles/database_backup/README.md
@@ -7,7 +7,7 @@ Generate backups for each build.
 database_backup:
   engines:
     - mysql
-
+  revert: true
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index 5b0d7289..b52413bf 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -7,6 +7,7 @@ Generate MySQL backups for each build.
 mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index bad0040f..4e371f28 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -11,7 +11,7 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -23,12 +23,45 @@ deploy_code:
   templates: []
   # Number of builds to keep. Note this is independant of databases/dump.
   keep: 10
-  # Wether to sync the local deploy base to a shared destination, after successful build.
+  # Whether to sync the local deploy base to a shared destination, after successful build.
   mount_sync: ""
   # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy"
+  # Type of file to use for sync - 'squashfs' or 'tarball'
+  # @see the _init role for SquashFS build dir paths
+  # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour
+  mount_type: "tarball"
   # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash.
   perms_fix_path: ""
   # perms_fix_path: "www/sites/default"
+  # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount.
+  # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
+  services: []
+  # services:
+  #   - php8.0-fpm
+  # What action to take against the services, 'reload' or 'stop'.
+  # Busy websites will require a hard stop of services to achieve the umount command.
+  service_action: reload
+  # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
+  rebuild_infra: false
+  # Details of API call to trigger. See api_call role.
+  api_call:
+    type: gitlab
+    base_url: https://gitlab.example.com/api/v4/
+    path: projects/1/ref/main/trigger/pipeline
+    method: POST
+    token: asdf-1234
+    token_type: trigger
+    variables: []
+    # example build parameters
+    #  - "[ENV]=dev"
+    #  - "[PLAY]=myserver.yml"
+    #  - "[RESOURCE]=myserver-example-com"
+    #  - "[REGION]=eu-west-1"
+    #  - "[EXTRA_PARAMS]=--force"
+    status_codes:
+      - 200
+      - 201
+      - 202
 
 ```
 
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml
index 0633bc75..ccc2c2cf 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml
@@ -1,5 +1,5 @@
 ---
-- include_tasks: "{{ maintenance_mode.operation }}.yml"
+- ansible.builtin.include_tasks: "{{ maintenance_mode.operation }}.yml"
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index cbe65b07..5fd4ddd0 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -1,8 +1,8 @@
 ---
-# @todo this needs refactoring for multisite.
 - name: Enable maintenance mode.
-  command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer"
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -11,8 +11,9 @@
     - project_type == 'drupal8'
 
 - name: Enable maintenance mode D7.
-  shell:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1"
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index e6a9cbf5..8337c968 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -1,7 +1,8 @@
 ---
 - name: Disable maintenance mode.
-  command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer"
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -9,8 +10,9 @@
     - project_type == 'drupal8'
 
 - name: Disable maintenance mode D7.
-  shell:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index bcb9c6ad..cf7e5311 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -5,6 +5,7 @@ Sync MySQL databases between environments.
 ```yaml
 ---
 mysql_sync:
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   databases:
     - source:
         # Name of the database to take a dump from.
@@ -20,6 +21,13 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
+        # Whether or not use to create a fresh database backup or use a nightly one.
+        fresh_db: true
+        # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default.
+        # This var is only used when fresh_db is set to "false".
+        dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
+        # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
@@ -30,6 +38,8 @@ mysql_sync:
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuilddev
+        # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty.
+        asg: ""
 
 ```
 

From cc31f6a261b71e913ee706d647958ad1b545715e Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 27 Jan 2023 14:28:15 +0100
Subject: [PATCH 036/139] Better deploy_code role docs. (#213)

* Better deploy_code role docs.

* roles path error in docs.

* roles path error in docs.

* Adding a note about deploy_previous handling for squashfs.
---
 docs/roles/deploy_code.md   | 80 ++++++++++++++++++++++++++++++++++++-
 roles/deploy_code/README.md | 80 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 158 insertions(+), 2 deletions(-)

diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index 4e371f28..30d87b11 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -1,5 +1,83 @@
 # Deploy
-Step that deploys the codebase.
+Step that deploys the codebase. On standalone machines and "static" clusters of web servers (e.g. machines whose addressing never changes) this is reasonably straightforward, the default variables should "just work". This role also supports deployment to autoscaling clusters of web servers, such as AWS autoscaling groups or containerised architecture. More details on that after this section.
+
+The shell script that wraps Ansible to handle the various build steps has various "stages" and the `deploy_code` role has a set of tasks for each stage. The key one for code building on the static/current cluster servers is [the `deploy.yml` file](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/deploy.yml). Here you will find the steps for checking out and building code on web servers, as well as the loading in of any application specific deploy code, [e.g. these tasks for Drupal 8](https://github.com/codeenigma/ce-deploy/tree/1.x/roles/deploy_code/deploy_code-drupal8/tasks). You choose what extra tasks to load via the `project_type` variable. Current core options are:
+
+* `drupal7`
+* `drupal8`
+* `matomo`
+* `mautic`
+* `simplesamlphp`
+
+Patches to support other common applications are always welcome! Also, Ansible inheritance being what it is, you can create your own custom deploy role in the same directory as your deployment playbook and Ansible will detect it and make it available to you. For example, if you create `./roles/deploy_code/deploy_code-myapp/tasks/main.yml` relative to your playbook and set `project_type: myapp` in your project variables then `ce-deploy` will load in those tasks.
+
+# Autoscale deployment
+For autoscaling clusters - no matter the underlying tech - the build code needs to be stored somewhere central and accessible to any potential new servers in the cluster. Because the performance of network attached storage (NAS) is often too poor or unreliable, we do not deploy the code to NAS - although this would be the simplest approach. Instead the technique we use is to build the code on each current server in the cluster, as though it were a static cluster or standalone machine, but *also* copy the code to the NAS so it is available to all future machines. This makes the existence of mounted NAS that is attached to all new servers a pre-requisite for `ce-deploy` to work with autoscaling.
+
+**Important**, autoscale deployments need to be carefully co-ordinated with [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) so new servers/containers have the correct scripts in place to place their code after they initialise. Specifically, the `mount_sync.tarballs` or `mount_sync.squashed_fs` list variables in `ce-provision` must contain paths that match with the location specified in the `deploy_code.mount_sync` variable in `ce-deploy` so `ce-deploy` copies code to the place `ce-provision`'s `cloud-init` scripts expect to find it. (More on the use of `cloud-init` below.)
+
+(An aside, we have previously supported S3-like object storage for storing the built code, but given all the applications we work with need to have NAS anyway for end user file uploads and other shared cluster resources, it seems pointless to introduce a second storage mechanism when we have one there already that works just fine.)
+
+This packaging of a copy of the code all happens in [the `cleanup.yml` file of the role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml). It supports three options:
+
+* No autoscale (or AWS AMI-based autoscale - see below) - leave `mount_sync` as an empty string
+* `tarball` type - makes a `tar.gz` with the code in and copies it to the NAS
+* `squashfs` type - packs a [`squashfs`](https://github.com/plougher/squashfs-tools) image, copies to the NAS and mounts it on each web server
+
+For both `tarball` and `squashfs` you need to set `mount_type` accordingly and the `mount_sync` variable to the location on your NAS where you want to store the built code.
+
+## `tarball` builds
+This is the simplest method of autoscale deployment, it simply packs up the code and copies it to the NAS at the end of the deployment. Everything else is just a standard "normal" build.
+
+**Important**, this method is only appropriate if you do not have too many files to deploy. The packing and restoring takes a very long time if there are many small files, so it is not appropriate for things like `composer` built PHP applications.
+
+### Rolling back
+With this method the live code directory is also the build directory, therefore you can edit the code in place in an emergency and "rolling back" if there are issues with a build is just a case of pointing the live build symlink back to the previous build. As long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the application. If the backup is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. By default this will be on the NAS so it is available to all web servers.
+
+## `squashfs` builds
+Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command).
+
+However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync)
+
+Consequently, at the build stage there are two important extra variables to set:
+
+```yaml
+deploy_code:
+  # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount.
+  # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
+  services: []
+  # services:
+  #   - php8.0-fpm
+  # What action to take against the services, 'reload' or 'stop'.
+  # Busy websites will require a hard stop of services to achieve the umount command.
+  service_action: reload
+```
+
+`services` is a list of Linux services to stop/reload in order to ensure the mount point is not locked. Usually this will be your PHP service, e.g.
+
+```yaml
+deploy_code:
+  services:
+    - php8.1-fpm
+```
+
+`service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time.
+
+Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory.
+
+### Rolling back
+Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event.
+
+Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it.
+
+Emergency code changes are possible but more fiddly. You have to copy the codebase from the mount to a sensible, *writeable* location, make your changes, [use the `squashfs` command to pack a new image](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L54), mount that image and, crucially, replace the `deploy.sqsh` image file on the NAS with your new image so future autoscale events will pick it up.
+
+# Autoscaling events
+Deploying code with autoscaling clusters relies on [cloud-init](https://cloudinit.readthedocs.io/) and is managed in our stack by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync). Whenever a new server spins up in a cluster, the `cloud-init` run-once script put in place by `ce-provision` is executed and that copies down the code from the NAS and deploys it to the correct location on the new server. At that point the server should become "healthy" and start serving the application.
+
+# AMI-based autoscale
+**This is experimental.** We are heavily based on [GitLab CE](https://gitlab.com/rluna-gitlab/gitlab-ce) and one of the options we support with [our provisioning tools](https://github.com/codeenigma/ce-provision/tree/1.x) is packing an [AWS AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the code embedded within, thus no longer requiring the [cloud-init](https://cloudinit.readthedocs.io/) step at all. [We call this option `repack` and the code is here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/aws/aws_ami/tasks/repack.yml) This makes provisioning of new machines in a cluster a little faster than the `squashfs` option, but requires the ability to trigger a build on our infrastructure `controller` server to execute a cluster build and pack the AMI. That is what the `api_call` dictionary below is providing for. You can see the API call constructed in [the last task of `cleanup.yml`](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L205).
+
 <!--TOC-->
 <!--ENDTOC-->
 
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index 4e371f28..30d87b11 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -1,5 +1,83 @@
 # Deploy
-Step that deploys the codebase.
+Step that deploys the codebase. On standalone machines and "static" clusters of web servers (e.g. machines whose addressing never changes) this is reasonably straightforward, the default variables should "just work". This role also supports deployment to autoscaling clusters of web servers, such as AWS autoscaling groups or containerised architecture. More details on that after this section.
+
+The shell script that wraps Ansible to handle the various build steps has various "stages" and the `deploy_code` role has a set of tasks for each stage. The key one for code building on the static/current cluster servers is [the `deploy.yml` file](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/deploy.yml). Here you will find the steps for checking out and building code on web servers, as well as the loading in of any application specific deploy code, [e.g. these tasks for Drupal 8](https://github.com/codeenigma/ce-deploy/tree/1.x/roles/deploy_code/deploy_code-drupal8/tasks). You choose what extra tasks to load via the `project_type` variable. Current core options are:
+
+* `drupal7`
+* `drupal8`
+* `matomo`
+* `mautic`
+* `simplesamlphp`
+
+Patches to support other common applications are always welcome! Also, Ansible inheritance being what it is, you can create your own custom deploy role in the same directory as your deployment playbook and Ansible will detect it and make it available to you. For example, if you create `./roles/deploy_code/deploy_code-myapp/tasks/main.yml` relative to your playbook and set `project_type: myapp` in your project variables then `ce-deploy` will load in those tasks.
+
+# Autoscale deployment
+For autoscaling clusters - no matter the underlying tech - the build code needs to be stored somewhere central and accessible to any potential new servers in the cluster. Because the performance of network attached storage (NAS) is often too poor or unreliable, we do not deploy the code to NAS - although this would be the simplest approach. Instead the technique we use is to build the code on each current server in the cluster, as though it were a static cluster or standalone machine, but *also* copy the code to the NAS so it is available to all future machines. This makes the existence of mounted NAS that is attached to all new servers a pre-requisite for `ce-deploy` to work with autoscaling.
+
+**Important**, autoscale deployments need to be carefully co-ordinated with [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) so new servers/containers have the correct scripts in place to place their code after they initialise. Specifically, the `mount_sync.tarballs` or `mount_sync.squashed_fs` list variables in `ce-provision` must contain paths that match with the location specified in the `deploy_code.mount_sync` variable in `ce-deploy` so `ce-deploy` copies code to the place `ce-provision`'s `cloud-init` scripts expect to find it. (More on the use of `cloud-init` below.)
+
+(An aside, we have previously supported S3-like object storage for storing the built code, but given all the applications we work with need to have NAS anyway for end user file uploads and other shared cluster resources, it seems pointless to introduce a second storage mechanism when we have one there already that works just fine.)
+
+This packaging of a copy of the code all happens in [the `cleanup.yml` file of the role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml). It supports three options:
+
+* No autoscale (or AWS AMI-based autoscale - see below) - leave `mount_sync` as an empty string
+* `tarball` type - makes a `tar.gz` with the code in and copies it to the NAS
+* `squashfs` type - packs a [`squashfs`](https://github.com/plougher/squashfs-tools) image, copies to the NAS and mounts it on each web server
+
+For both `tarball` and `squashfs` you need to set `mount_type` accordingly and the `mount_sync` variable to the location on your NAS where you want to store the built code.
+
+## `tarball` builds
+This is the simplest method of autoscale deployment, it simply packs up the code and copies it to the NAS at the end of the deployment. Everything else is just a standard "normal" build.
+
+**Important**, this method is only appropriate if you do not have too many files to deploy. The packing and restoring takes a very long time if there are many small files, so it is not appropriate for things like `composer` built PHP applications.
+
+### Rolling back
+With this method the live code directory is also the build directory, therefore you can edit the code in place in an emergency and "rolling back" if there are issues with a build is just a case of pointing the live build symlink back to the previous build. As long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the application. If the backup is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. By default this will be on the NAS so it is available to all web servers.
+
+## `squashfs` builds
+Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command).
+
+However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync)
+
+Consequently, at the build stage there are two important extra variables to set:
+
+```yaml
+deploy_code:
+  # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount.
+  # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted.
+  services: []
+  # services:
+  #   - php8.0-fpm
+  # What action to take against the services, 'reload' or 'stop'.
+  # Busy websites will require a hard stop of services to achieve the umount command.
+  service_action: reload
+```
+
+`services` is a list of Linux services to stop/reload in order to ensure the mount point is not locked. Usually this will be your PHP service, e.g.
+
+```yaml
+deploy_code:
+  services:
+    - php8.1-fpm
+```
+
+`service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time.
+
+Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory.
+
+### Rolling back
+Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event.
+
+Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it.
+
+Emergency code changes are possible but more fiddly. You have to copy the codebase from the mount to a sensible, *writeable* location, make your changes, [use the `squashfs` command to pack a new image](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L54), mount that image and, crucially, replace the `deploy.sqsh` image file on the NAS with your new image so future autoscale events will pick it up.
+
+# Autoscaling events
+Deploying code with autoscaling clusters relies on [cloud-init](https://cloudinit.readthedocs.io/) and is managed in our stack by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync). Whenever a new server spins up in a cluster, the `cloud-init` run-once script put in place by `ce-provision` is executed and that copies down the code from the NAS and deploys it to the correct location on the new server. At that point the server should become "healthy" and start serving the application.
+
+# AMI-based autoscale
+**This is experimental.** We are heavily based on [GitLab CE](https://gitlab.com/rluna-gitlab/gitlab-ce) and one of the options we support with [our provisioning tools](https://github.com/codeenigma/ce-provision/tree/1.x) is packing an [AWS AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the code embedded within, thus no longer requiring the [cloud-init](https://cloudinit.readthedocs.io/) step at all. [We call this option `repack` and the code is here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/aws/aws_ami/tasks/repack.yml) This makes provisioning of new machines in a cluster a little faster than the `squashfs` option, but requires the ability to trigger a build on our infrastructure `controller` server to execute a cluster build and pack the AMI. That is what the `api_call` dictionary below is providing for. You can see the API call constructed in [the last task of `cleanup.yml`](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L205).
+
 <!--TOC-->
 <!--ENDTOC-->
 

From f6a3cb362eae9bb05d9d2dee3f8cd41f7ddcebae Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 27 Jan 2023 19:42:04 +0200
Subject: [PATCH 037/139] Cachetool settings and drupal7 tasks fix pr 1.x
 (#211)

* cachetool_settings_and_drupal7_tasks_fix

* cachetool_settings_and_drupal7_tasks_fix

* cachetool_settings_and_drupal7_tasks_fix

* cachetool settings fixing condition and namespace

* fixing cachetool adapter options

* fixed var

* fixed var

* fixed var
---
 .../cache_clear-opcache/defaults/main.yml     |  2 +-
 .../cache_clear-opcache/tasks/main.yml        | 16 +++--
 roles/cli/cachetool/defaults/main.yml         |  2 +-
 roles/cli/cachetool/tasks/main.yml            | 68 ++++++++++++++++---
 .../tasks/offline.yml                         |  2 +-
 .../tasks/online.yml                          |  2 +-
 6 files changed, 76 insertions(+), 16 deletions(-)

diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
index 8121226e..97623e62 100644
--- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
@@ -4,7 +4,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  adapter: ""
+  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
index bee9e6fa..57feac51 100644
--- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
@@ -1,16 +1,24 @@
 ---
+- name: Get latest php installed
+  ansible.builtin.shell:
+    cmd: "ls /etc/php/ | tail -1"
+  register: _php_version
+
+- name: Set cachetool adapter.
+  ansible.builtin.set_fact:
+    _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('--fcgi=127.0.0.1:90' + _php_version.stdout | replace('.','')) }}"
 
 - name: Clear opcache.
   command:
-    cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n opcache:reset"
+    cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset"
   when: cache_clear_opcache.clear_opcache
 
 - name: Clear apcu.
   command:
-    cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n apcu:cache:clear all"
+    cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear all"
   when: cache_clear_opcache.clear_apcu
 
 - name: Clear stats.
   command:
-    cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n stat:clear"
-  when: cache_clear_opcache.clear_stat
\ No newline at end of file
+    cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear"
+  when: cache_clear_opcache.clear_stat
diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml
index 5ec1b2d7..8b98934d 100644
--- a/roles/cli/cachetool/defaults/main.yml
+++ b/roles/cli/cachetool/defaults/main.yml
@@ -1,3 +1,3 @@
 ---
 cachetool:
-  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
\ No newline at end of file
+  version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version.
diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index aa27df04..12cba942 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -13,22 +13,74 @@
   when:
     - deploy_operation == 'deploy'
 
-- name: Download latest cachetool installer.
-  get_url:
-    url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar"
-    dest: "{{ cachetool_bin }}"
-    mode: 0755
+- name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility
+  block:
+    - name: Get latest php installed
+      ansible.builtin.shell:
+        cmd: "ls /etc/php/ | tail -1 | sed -e 's/\\.//g'"
+      register: _php_version_str
+
+    - name: Download latest cachetool installer if PHP is 8.1 or newer.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int >= 81
+
+    - name: Download cachetool version 8.5.0 installer if PHP is 8.0.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int == 80
+
+    - name: Download cachetool version 7.1.0 installer if PHP is 7.3 or newer.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int >= 73
+        - _php_version_str.stdout | int < 80
+
+    - name: Download cachetool version 5.1.3 installer if PHP is 7.2.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int == 72
+
+    - name: Download cachetool version 4.1.1 installer if PHP is 7.1.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int == 71
+
+    - name: Download cachetool version 3.2.2 installer if PHP version is too old.
+      ansible.builtin.get_url:
+        url: "http://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar"
+        dest: "{{ cachetool_bin }}"
+        mode: 0755
+      when:
+        - _php_version_str.stdout | int < 71
+
   when:
     - deploy_operation == 'deploy'
     - not cachetool_global.stat.exists
-    - cachetool.version == 'latest'
+    - cachetool.version | length == 0
 
 - name: "Download cachetool version {{ cachetool.version }} installer."
-  get_url:
+  ansible.builtin.get_url:
     url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
     dest: "{{ cachetool_bin }}"
     mode: 0755
   when:
     - deploy_operation == 'deploy'
     - not cachetool_global.stat.exists
-    - cachetool.version != 'latest'
+    - cachetool.version is defined
+    - cachetool.version | length > 0
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index 5fd4ddd0..f8ae276c 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -12,7 +12,7 @@
 
 - name: Enable maintenance mode D7.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1"
   args:
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 8337c968..dfc3e3b3 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -11,7 +11,7 @@
 
 - name: Disable maintenance mode D7.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
   args:
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"

From 44fef49398aec66a187d7c05c00d53ba9023cf6b Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 31 Jan 2023 13:02:14 +0200
Subject: [PATCH 038/139] previous_cachetool_remove_before_install_new (#224)

* previous_cachetool_remove_before_install_new

* namespace fix
---
 roles/cli/cachetool/tasks/main.yml | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index 12cba942..2e042957 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -1,17 +1,8 @@
 ---
-- name: Check if we already have cachetool.
-  stat:
+- name: Remove previous cachetool if exists.
+  ansible.builtin.file:
     path: "{{ cachetool_bin }}"
-  register: cachetool_global
-  when:
-    - deploy_operation == 'deploy'
-
-- name: Ensure bin directory exists.
-  file:
-    path: "{{ cachetool_bin | dirname }}"
-    state: directory
-  when:
-    - deploy_operation == 'deploy'
+    state: absent
 
 - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility
   block:
@@ -71,7 +62,6 @@
 
   when:
     - deploy_operation == 'deploy'
-    - not cachetool_global.stat.exists
     - cachetool.version | length == 0
 
 - name: "Download cachetool version {{ cachetool.version }} installer."
@@ -81,6 +71,5 @@
     mode: 0755
   when:
     - deploy_operation == 'deploy'
-    - not cachetool_global.stat.exists
     - cachetool.version is defined
     - cachetool.version | length > 0

From f1b2b902c80981fed27458a7640a6ddbfdae15ba Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 31 Jan 2023 14:24:16 +0200
Subject: [PATCH 039/139] Fix cachetool removal pr 1.x (#227)

* fix_cachetool_removal_step

* fix task name
---
 roles/cli/cachetool/tasks/main.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index 2e042957..499b9ebd 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -3,6 +3,8 @@
   ansible.builtin.file:
     path: "{{ cachetool_bin }}"
     state: absent
+  when:
+    - deploy_operation == 'deploy'
 
 - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility
   block:
@@ -64,7 +66,7 @@
     - deploy_operation == 'deploy'
     - cachetool.version | length == 0
 
-- name: "Download cachetool version {{ cachetool.version }} installer."
+- name: "Download the specified {{ cachetool.version }} cachetool version installer."
   ansible.builtin.get_url:
     url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
     dest: "{{ cachetool_bin }}"

From de6d80bdb30ef46ec5dd716a0781b35b1432c5cd Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Wed, 1 Feb 2023 17:29:08 +0200
Subject: [PATCH 040/139] Fix cachetool removal pr 1.x (#229)

* fix_cachetool_removal_step

* fix task name

* fix_cachetool_removal_task

* fix_cachetool_removal_task
---
 roles/cli/cachetool/tasks/main.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index 499b9ebd..f0f05b28 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -6,6 +6,13 @@
   when:
     - deploy_operation == 'deploy'
 
+- name: Ensure bin directory exists.
+  ansible.builtin.file:
+    path: "{{ cachetool_bin | dirname }}"
+    state: directory
+  when:
+    - deploy_operation == 'deploy'
+
 - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility
   block:
     - name: Get latest php installed

From 1eda678122c41d5e3da08a03f8e70f6fd0f865ba Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 3 Feb 2023 12:45:36 +0100
Subject: [PATCH 041/139] Documentation enhancements pr 1.x (#218)

* Better deploy_code role docs.

* roles path error in docs.

* roles path error in docs.

* Adding a note about deploy_previous handling for squashfs.

* Reference incorrect role for deploy user sudo perms.

* Minor edits to frontpage README.

* Rebuilt docs.

* Accidentally overwrote docs change.
---
 README.md                                           |  5 ++---
 docs/roles/cache_clear/cache_clear-opcache.md       |  2 +-
 docs/roles/cli/cachetool.md                         |  3 ++-
 docs/roles/deploy_code.md                           |  6 +++---
 roles/cache_clear/cache_clear-opcache/README.md     |  2 +-
 roles/cli/cachetool/README.md                       |  3 ++-
 roles/deploy_code/README.md                         |  6 +++---
 .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 13 ++++++++-----
 .../admin_creds/admin_creds-drupal8/tasks/admin.yml |  6 +++---
 9 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 41910905..3046acb1 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,10 @@
 # ce-deploy
 
-[![Build Status](https://api.travis-ci.com/codeenigma/ce-deploy.svg?branch=1.x)](https://api.travis-ci.com/codeenigma/ce-deploy.svg?branch=1.x)
-
 A set of Ansible roles and wrapper scripts to deploy (web) applications.
+
 ## Overview
 The "stack" from this repo is to be installed on a "deploy" server/runner, to be used in conjonction with a CI/CD tool (Jenkins, Gitlab, Travis, ...).
-It allows the deploy steps for a given app to be easily customizable at will, and to be stored alongside the codebase of the project.
+It allows the deploy steps for a given app to be easily customisable and to be stored alongside the codebase of the project.
 When triggered from a deployment tool, the stack will clone the codebase and "play" a given deploy playbook from there.
 
 <!--TOC-->
diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md
index e93a1c33..c7959bcb 100644
--- a/docs/roles/cache_clear/cache_clear-opcache.md
+++ b/docs/roles/cache_clear/cache_clear-opcache.md
@@ -14,7 +14,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  adapter: ""
+  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
diff --git a/docs/roles/cli/cachetool.md b/docs/roles/cli/cachetool.md
index 9cba4a32..36d0a014 100644
--- a/docs/roles/cli/cachetool.md
+++ b/docs/roles/cli/cachetool.md
@@ -5,7 +5,8 @@ Installs the `drush` command-line tool for the deploy user.
 ```yaml
 ---
 cachetool:
-  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
+  version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version.
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index 30d87b11..377ca80c 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -37,7 +37,7 @@ With this method the live code directory is also the build directory, therefore
 ## `squashfs` builds
 Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command).
 
-However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync)
+However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `squashfs` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/squashfs).
 
 Consequently, at the build stage there are two important extra variables to set:
 
@@ -63,10 +63,10 @@ deploy_code:
 
 `service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time.
 
-Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory.
+Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is named `PROJECTNAME_BUILDTYPE_deploy.sqsh`. [See the code here.](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L80) The previous codebase is *also* packed and copied to the NAS, named `PROJECTNAME_BUILDTYPE_deploy_previous.sqsh` in the same directory.
 
 ### Rolling back
-Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event.
+Rolling back from a bad `squashfs` build means copying the project/branch specific `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting the `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename your `deploy_previous.sqsh` as `PROJECTNAME_BUILDTYPE_deploy.sqsh`, so it is used on an autoscale event. [You can see the actual code that runs on autoscale events here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/mount_sync/templates/init-squashfs.sh.j2)
 
 Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it.
 
diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md
index e93a1c33..c7959bcb 100644
--- a/roles/cache_clear/cache_clear-opcache/README.md
+++ b/roles/cache_clear/cache_clear-opcache/README.md
@@ -14,7 +14,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  adapter: ""
+  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
diff --git a/roles/cli/cachetool/README.md b/roles/cli/cachetool/README.md
index 9cba4a32..36d0a014 100644
--- a/roles/cli/cachetool/README.md
+++ b/roles/cli/cachetool/README.md
@@ -5,7 +5,8 @@ Installs the `drush` command-line tool for the deploy user.
 ```yaml
 ---
 cachetool:
-  version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version
+  version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version.
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index 30d87b11..377ca80c 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -37,7 +37,7 @@ With this method the live code directory is also the build directory, therefore
 ## `squashfs` builds
 Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command).
 
-However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync)
+However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `squashfs` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/squashfs).
 
 Consequently, at the build stage there are two important extra variables to set:
 
@@ -63,10 +63,10 @@ deploy_code:
 
 `service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time.
 
-Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory.
+Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is named `PROJECTNAME_BUILDTYPE_deploy.sqsh`. [See the code here.](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L80) The previous codebase is *also* packed and copied to the NAS, named `PROJECTNAME_BUILDTYPE_deploy_previous.sqsh` in the same directory.
 
 ### Rolling back
-Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event.
+Rolling back from a bad `squashfs` build means copying the project/branch specific `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting the `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename your `deploy_previous.sqsh` as `PROJECTNAME_BUILDTYPE_deploy.sqsh`, so it is used on an autoscale event. [You can see the actual code that runs on autoscale events here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/mount_sync/templates/init-squashfs.sh.j2)
 
 Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it.
 
diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
index c8d75bd7..22f9e55e 100644
--- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
+++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
@@ -1,11 +1,14 @@
 ---
-
-- set_fact:
+- ansible.builtin.set_fact:
     _admin_user: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-user chars=ascii_letters') }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}"
 
 - name: Reset admin username.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\""
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\""
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
 - name: Reset admin password.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'"
\ No newline at end of file
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
index c2adafdf..dad5955a 100644
--- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
+++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml
@@ -1,16 +1,16 @@
 ---
 # For some unknown reason, updating the field_user_data table did not work reliably,
 # nor did drush upwd for the password.
-# Loading the user directly is akward, but at least means we don't bypass entity update.
+# Loading the user directly is awkward, but at least means we don't bypass entity update.
 - name: Reset admin username.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' "
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when:
     - admin_creds.username | length > 1
 
 - name: Reset admin password.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' "
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when:

From 09e86396d984ed6d8fbe79142c9bf373ec39eac2 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 3 Feb 2023 12:56:57 +0100
Subject: [PATCH 042/139] Drush refactor pr 1.x (#231)

* Fixing sync jobs.

* Fixing bad task title.
---
 .../database_apply-drupal7/tasks/ctools.yml   |  8 +++--
 .../database_sync-mysql/tasks/main.yml        |  2 +-
 .../database_sync-mysql/tasks/sync.yml        | 32 ++++++++++---------
 roles/sync/database_sync/tasks/main.yml       |  2 +-
 .../cache_clear-drupal8/tasks/main.yml        | 13 ++++++++
 .../database_apply-drupal7/tasks/main.yml     |  2 +-
 .../database_apply-drupal8/tasks/main.yml     | 13 ++++++++
 roles/sync/files_sync/tasks/main.yml          |  2 +-
 8 files changed, 53 insertions(+), 21 deletions(-)

diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
index ce4db921..194226fa 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
@@ -1,8 +1,12 @@
 ---
 - name: Check if Ctools module is enabled.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} &&{{ drush_bin }}pm-info ctools | grep ':  enabled' | wc -l"
+  ansible.builtin.shell:
+    cmd: "{{ drush_bin }} pm-info ctools | grep ':  enabled' | wc -l"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   register: ctools_enabled
 
 - name: Revert Drupal configuration from Ctools.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} &&{{ drush_bin }}-y ctools-export-revert --all"
+  ansible.builtin.command:
+    cmd: "{{ drush_bin }} -y ctools-export-revert --all"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when: ctools_enabled.stdout == "1"
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/main.yml b/roles/sync/database_sync/database_sync-mysql/tasks/main.yml
index 11a47d5a..63edc494 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Sync database.
-  include_tasks: "sync.yml"
+  ansible.builtin.include_tasks: "sync.yml"
   with_items: "{{ mysql_sync.databases }}"
   loop_control:
     loop_var: database
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 98eca0f3..29100ab5 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -36,11 +36,11 @@
     - database.target.asg | length > 0
 
 - name: Register remote dump name (from database).
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2"
 
 - name: Get source last known good build number.
-  command:
+  ansible.builtin.command:
     argv:
       - "/bin/sh"
       - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh"
@@ -51,17 +51,17 @@
   when: database.source.type == 'rolling'
 
 - name: Register source database name.
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}"
   when: database.source.type == 'rolling'
 
 - name: Register source database name.
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_source_database: "{{ database.source.database }}"
   when: not database.source.type == 'rolling'
 
 - name: Take a dump from source database.
-  shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
   delegate_to: "{{ database.source.host }}"
   when:
     - database.source.fresh_db
@@ -88,11 +88,11 @@
   when: not database.source.fresh_db
 
 - name: Register tmp target dump name.
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2"
 
 - name: Get target last known good build number.
-  command:
+  ansible.builtin.command:
     argv:
       - "/bin/sh"
       - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh"
@@ -103,38 +103,40 @@
   when: database.target.type == 'rolling'
 
 - name: Register target database name.
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}"
   when: database.target.type == 'rolling'
 
 - name: Register target database name.
-  set_fact:
+  ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}"
   when: not database.target.type == 'rolling'
 
 - name: Fetch dump file.
-  fetch:
+  ansible.builtin.fetch:
     src: "{{ mysql_sync_source_dump_path }}"
     dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
     flat: true
   delegate_to: "{{ database.source.host }}"
 
 - name: Copy dump file to destination.
-  copy:
+  ansible.builtin.copy:
     src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
     dest: "{{ mysql_sync_target_dump_path }}"
 
 - name: Drop target database.
-  shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'"
+  ansible.builtin.command:
+    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'"
 
 - name: Recreate target database.
-  shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
+  ansible.builtin.command:
+    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
 
 - name: Repopulate database from dump.
-  shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
+  ansible.builtin.shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
 
 - name: Remove tmp dump file.
-  file:
+  ansible.builtin.file:
     path: "{{ mysql_sync_target_dump_path }}"
     state: absent
 
diff --git a/roles/sync/database_sync/tasks/main.yml b/roles/sync/database_sync/tasks/main.yml
index a3992186..650a9e0a 100644
--- a/roles/sync/database_sync/tasks/main.yml
+++ b/roles/sync/database_sync/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 
 - name: Sync databases.
-  include_role:
+  ansible.builtin.include_role:
     name: "sync/database_sync/database_sync-{{ engine }}"
   with_items: "{{ database_sync.engines }}"
   loop_control:
diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
index 96c721ee..431366cf 100644
--- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,4 +1,13 @@
 ---
+# Drupal 8 ships drush with the website code so we need the previous build in the path.
+- name: Stash the drush_bin variable.
+  ansible.builtin.set_fact:
+    _drush_bin_deploy: "{{ drush_bin }}"
+
+- name: Update location of drush for the sync cache clear command.
+  ansible.builtin.set_fact:
+    drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush"
+
 - name: Clear Drupal cache.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
@@ -9,3 +18,7 @@
   loop_control:
     loop_var: site
   run_once: true
+
+- name: Restore the drush_bin variable.
+  ansible.builtin.set_fact:
+    drush_bin: "{{ _drush_bin_deploy }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
index f520514e..ea4c083b 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Apply Drupal database updates.
-  ansible.builtin.shell:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   with_items: "{{ drupal.sites }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
index e050af3f..96d9b65f 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
@@ -1,4 +1,13 @@
 ---
+# Drupal 8 ships drush with the website code so we need the previous build in the path.
+- name: Stash the drush_bin variable.
+  ansible.builtin.set_fact:
+    _drush_bin_deploy: "{{ drush_bin }}"
+
+- name: Update location of drush for the update/config import commands.
+  ansible.builtin.set_fact:
+    drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush"
+
 - name: Apply Drupal database updates.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
@@ -26,3 +35,7 @@
 - name: Clear the cache.
   ansible.builtin.include_role:
     name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}"
+
+- name: Restore the drush_bin variable.
+  ansible.builtin.set_fact:
+    drush_bin: "{{ _drush_bin_deploy }}"
diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml
index 47c3667a..3107eb24 100644
--- a/roles/sync/files_sync/tasks/main.yml
+++ b/roles/sync/files_sync/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Sync files.
-  include_tasks: "sync.yml"
+  ansible.builtin.include_tasks: "sync.yml"
   with_items: "{{ files_sync.directories }}"
   loop_control:
     loop_var: files

From 3ba2fe09b1004798dce2feee62619b1eafda0409 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 17 Feb 2023 12:54:40 +0100
Subject: [PATCH 043/139] Drush refactor pr 1.x (#234)

* Fixing sync jobs.

* Fixing bad task title.

* Trying to fix D7 drush issues with chdir.
---
 .../maintenance_mode-drupal-core/tasks/offline.yml            | 4 ++--
 .../maintenance_mode-drupal-core/tasks/online.yml             | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index f8ae276c..2d6a9310 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -10,10 +10,10 @@
     - previous_build_number > 0
     - project_type == 'drupal8'
 
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Enable maintenance mode D7.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1"
-  args:
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index dfc3e3b3..6b127f89 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -9,10 +9,10 @@
   when:
     - project_type == 'drupal8'
 
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Disable maintenance mode D7.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
-  args:
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"

From 75ff9eedf93c0b438d4c33b9775275121fcb8045 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 17 Feb 2023 13:20:10 +0100
Subject: [PATCH 044/139] Drush refactor pr 1.x (#236)

* Fixing sync jobs.

* Fixing bad task title.

* Trying to fix D7 drush issues with chdir.

* Moving D7 drush commands back to 'shell'.
---
 .../database_apply-drupal7/tasks/ctools.yml          |  3 ++-
 .../database_apply-drupal7/tasks/features.yml        |  5 +++--
 .../database_apply-drupal7/tasks/main.yml            | 12 ++++++------
 .../admin_creds/admin_creds-drupal7/tasks/admin.yml  |  5 +++--
 .../cache_clear/cache_clear-drupal7/tasks/main.yml   |  3 ++-
 .../database_apply-drupal7/tasks/main.yml            |  3 ++-
 6 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
index 194226fa..8102d3d6 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
@@ -1,4 +1,5 @@
 ---
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Check if Ctools module is enabled.
   ansible.builtin.shell:
     cmd: "{{ drush_bin }} pm-info ctools | grep ':  enabled' | wc -l"
@@ -6,7 +7,7 @@
   register: ctools_enabled
 
 - name: Revert Drupal configuration from Ctools.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -y ctools-export-revert --all"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when: ctools_enabled.stdout == "1"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml
index edba46bf..f55795dc 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/features.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml
@@ -1,8 +1,9 @@
 ---
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Check if Features module is enabled.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ':  enabled' | wc -l"
+  ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ':  enabled' | wc -l"
   register: features_enabled
 
 - name: Revert Drupal configuration from Features.
-  shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}"
+  ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}"
   when: features_enabled.stdout == "1"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index fe25bcaa..70b21947 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
-
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Install Drupal.
-  shell:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
@@ -12,7 +12,7 @@
   when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
 
 - name: Fix permissions on Drupal directory.
-  shell:
+  ansible.builtin.shell:
     cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   with_items: "{{ drupal.sites }}"
   loop_control:
@@ -30,7 +30,7 @@
   when: previous_build_number > 0
 
 - name: Apply Drupal database updates.
-  shell:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   with_items: "{{ drupal.sites }}"
@@ -38,7 +38,7 @@
     loop_var: site
 
 - name: Revert Drupal configuration with Features.
-  include_tasks: features.yml
+  ansible.builtin.include_tasks: features.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
@@ -47,7 +47,7 @@
     - site.revert_features_command
 
 - name: Revert Drupal configuration with Ctools.
-  include_tasks: ctools.yml
+  ansible.builtin.include_tasks: ctools.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
index 22f9e55e..add864cf 100644
--- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
+++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml
@@ -4,11 +4,12 @@
 - ansible.builtin.set_fact:
     _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}"
 
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Reset admin username.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\""
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
 - name: Reset admin password.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
index ab9342ba..fd22fbff 100644
--- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml
@@ -1,6 +1,7 @@
 ---
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Clear Drupal 7 cache.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
index ea4c083b..eeaa2d73 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml
@@ -1,6 +1,7 @@
 ---
+# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Apply Drupal database updates.
-  ansible.builtin.command:
+  ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   with_items: "{{ drupal.sites }}"

From 8d0226057a6c979856dfd1a2d44110b1cb1f3a91 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Mon, 6 Mar 2023 19:27:43 +0200
Subject: [PATCH 045/139] Change php detection for fastcgi and cachetool pr 1.x
 (#238)

* change_php_detection_for_fastcgi_and_cachetool

* fix_typo

* change_php_detection_for_fastcgi_and_cachetool_no_sudo
---
 roles/cache_clear/cache_clear-opcache/tasks/main.yml | 2 +-
 roles/cli/cachetool/tasks/main.yml                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
index 57feac51..d7c9f6e2 100644
--- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
@@ -1,7 +1,7 @@
 ---
 - name: Get latest php installed
   ansible.builtin.shell:
-    cmd: "ls /etc/php/ | tail -1"
+    cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1'
   register: _php_version
 
 - name: Set cachetool adapter.
diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index f0f05b28..2e2a49d1 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -17,7 +17,7 @@
   block:
     - name: Get latest php installed
       ansible.builtin.shell:
-        cmd: "ls /etc/php/ | tail -1 | sed -e 's/\\.//g'"
+        cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"'
       register: _php_version_str
 
     - name: Download latest cachetool installer if PHP is 8.1 or newer.

From 7971ce73325610df48987a1adc4ac196a2b05a30 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 14 Mar 2023 13:45:43 +0200
Subject: [PATCH 046/139] Cachetool bin per site pr 1.x (#243)

* cachetool_bin_per_project

* cachetool_bin_per_site_fix

* cachetool_bin_per_site_fix2
---
 roles/_init/tasks/main.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index effa2790..e939def3 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -107,7 +107,15 @@
 
 - name: Define opcache cachetool path.
   ansible.builtin.set_fact:
-    cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}"
+    cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}"
+  when: deploy_code.mount_type != "squashfs"
+
+- name: Define opcache cachetool path if SquashFS deploy.
+  ansible.builtin.set_fact:
+    cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 
 - name: Ensure we have a cachetool binary.
   ansible.builtin.import_role:

From 31c3197a50a0cba48e5634499b914915bffb7640 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 14 Mar 2023 18:26:40 +0100
Subject: [PATCH 047/139] apcu no longer has an 'all' option. (#245)

---
 roles/cache_clear/cache_clear-opcache/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
index d7c9f6e2..428e9053 100644
--- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
@@ -15,7 +15,7 @@
 
 - name: Clear apcu.
   command:
-    cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear all"
+    cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear"
   when: cache_clear_opcache.clear_apcu
 
 - name: Clear stats.

From b5fed246e214332b105cd2759cea91be2743b7e8 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 15 Mar 2023 10:21:40 +0100
Subject: [PATCH 048/139] Refactoring cachetool path setting to match other
 squashfs vars. (#247)

---
 roles/_init/tasks/main.yml | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index e939def3..7f28f3ff 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -21,6 +21,9 @@
 - name: Define live_symlink dest.
   ansible.builtin.set_fact:
     live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
+- name: Define opcache cachetool path.
+  ansible.builtin.set_fact:
+    cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}"
 
 # Manipulate variables for SquashFS builds.
 - name: Define image builds base path.
@@ -47,6 +50,12 @@
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
+- name: Overwrite cachetool path if SquashFS deploy.
+  ansible.builtin.set_fact:
+    cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 
 # Gather last known good build directly from symlink.
 # This can happen:
@@ -105,18 +114,6 @@
   when:
     - _project_type_task_result.stat.exists
 
-- name: Define opcache cachetool path.
-  ansible.builtin.set_fact:
-    cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}"
-  when: deploy_code.mount_type != "squashfs"
-
-- name: Define opcache cachetool path if SquashFS deploy.
-  ansible.builtin.set_fact:
-    cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}"
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-
 - name: Ensure we have a cachetool binary.
   ansible.builtin.import_role:
     name: cli/cachetool

From 459689f9462f3e6087f14abadfb54d575a129ff6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 15 Mar 2023 12:58:44 +0100
Subject: [PATCH 049/139] Cachetool pr 1.x (#249)

* Refactoring cachetool path setting to match other squashfs vars.

* Refactoring cachetool_bin handling.
---
 docs/roles/cache_clear/cache_clear-opcache.md          |  1 +
 roles/_init/tasks/main.yml                             | 10 +++++++++-
 roles/cache_clear/cache_clear-opcache/README.md        |  1 +
 .../cache_clear/cache_clear-opcache/defaults/main.yml  |  1 +
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md
index c7959bcb..465307e2 100644
--- a/docs/roles/cache_clear/cache_clear-opcache.md
+++ b/docs/roles/cache_clear/cache_clear-opcache.md
@@ -19,6 +19,7 @@ cache_clear_opcache:
   clear_opcache: true
   clear_apcu: false
   clear_stat: false
+  # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined
 
 ```
 
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 7f28f3ff..f68be544 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -23,7 +23,14 @@
     live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
 - name: Define opcache cachetool path.
   ansible.builtin.set_fact:
-    cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}"
+    cachetool_bin: "{{ deploy_base_path }}/cachetool.phar"
+  when: cache_clear_opcache.cachetool_bin is not defined
+- name: Set opcache cachetool path from variable.
+  ansible.builtin.set_fact:
+    cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}"
+  when:
+    - cache_clear_opcache.cachetool_bin is defined
+    - cache_clear_opcache.cachetool_bin | length > 0
 
 # Manipulate variables for SquashFS builds.
 - name: Define image builds base path.
@@ -56,6 +63,7 @@
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
+    - cache_clear_opcache.cachetool_bin is not defined
 
 # Gather last known good build directly from symlink.
 # This can happen:
diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md
index c7959bcb..465307e2 100644
--- a/roles/cache_clear/cache_clear-opcache/README.md
+++ b/roles/cache_clear/cache_clear-opcache/README.md
@@ -19,6 +19,7 @@ cache_clear_opcache:
   clear_opcache: true
   clear_apcu: false
   clear_stat: false
+  # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined
 
 ```
 
diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
index 97623e62..222325ce 100644
--- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
@@ -9,3 +9,4 @@ cache_clear_opcache:
   clear_opcache: true
   clear_apcu: false
   clear_stat: false
+  # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined

From 0b39b64f6e6fe3c430943a54f04cd072a6689a56 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 15 Mar 2023 13:00:25 +0100
Subject: [PATCH 050/139] Ensuring we can prevent features and ctools
 deployments in Drupal 7. (#240)

---
 roles/_init/defaults/main.yml                              | 3 ++-
 roles/database_apply/database_apply-drupal7/tasks/main.yml | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index b38c1e59..00d03c05 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -14,6 +14,7 @@ drupal:
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
+      revert_ctools: true
       # End Drupal 7 variables
       sanitize_command: "sql-sanitize"
       base_url: https://www.example.com
@@ -25,4 +26,4 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-bin_directory: "/home/{{ deploy_user }}/.bin"
\ No newline at end of file
+bin_directory: "/home/{{ deploy_user }}/.bin"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 70b21947..6b745c3b 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -44,10 +44,12 @@
     loop_var: site
   when:
     - previous_build_number > 0
-    - site.revert_features_command
+    - site.revert_features_command | length > 0
 
 - name: Revert Drupal configuration with Ctools.
   ansible.builtin.include_tasks: ctools.yml
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
+  when:
+    - site.revert_ctools

From 3d6172aca2d37d6d3109192030f1f77d45b62e3d Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 15 Mar 2023 13:21:48 +0100
Subject: [PATCH 051/139] Cachetool pr 1.x (#251)

* Refactoring cachetool path setting to match other squashfs vars.

* Refactoring cachetool_bin handling.

* Forgot to remove the default() filter from squashfs var setting.

* Refactoring clauses slightly to simplify.
---
 roles/_init/tasks/main.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index f68be544..a0d90571 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -24,8 +24,7 @@
 - name: Define opcache cachetool path.
   ansible.builtin.set_fact:
     cachetool_bin: "{{ deploy_base_path }}/cachetool.phar"
-  when: cache_clear_opcache.cachetool_bin is not defined
-- name: Set opcache cachetool path from variable.
+- name: Set opcache cachetool path from variable if provided.
   ansible.builtin.set_fact:
     cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}"
   when:
@@ -57,9 +56,9 @@
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
-- name: Overwrite cachetool path if SquashFS deploy.
+- name: Overwrite cachetool path if SquashFS deploy and path not provided.
   ansible.builtin.set_fact:
-    cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}"
+    cachetool_bin: "{{ build_base_path }}/cachetool.phar"
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"

From 1f2e3b66bb060ec7123ccc253eab2fb487b3e588 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Mon, 20 Mar 2023 18:13:36 +0200
Subject: [PATCH 052/139] improving_old_builds_cleanup (#254)

* improving_old_builds_cleanup

* improving_old_builds_cleanup_fix
---
 roles/_init/defaults/main.yml                          |  2 ++
 .../database_backup-mysql/tasks/cleanup-dump.yml       |  2 +-
 .../database_backup-mysql/tasks/cleanup-rolling.yml    |  3 +--
 .../database_backup-mysql/tasks/cleanup.yml            |  3 +--
 roles/deploy_code/tasks/cleanup.yml                    | 10 +++++-----
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 00d03c05..971f1b9b 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -27,3 +27,5 @@ mautic:
   image_path: "media/images"
   force_install: false
 bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
index a9ea9ae2..2609cabe 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
@@ -4,5 +4,5 @@
   ansible.builtin.file:
     path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2"
     state: absent
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
index 2bdf29ee..374e65ca 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
@@ -1,5 +1,4 @@
 ---
-# We assume it's safe to only go back 50 build back.
 - name: Delete mysql databases.
   community.mysql.mysql_db:
     name: "{{ database.database }}_{{ item }}"
@@ -7,5 +6,5 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
index ac71fcbb..89cbf171 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
@@ -15,7 +15,6 @@
 
 - ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml"
 
-# We assume it's safe to only go back 50 build back.
 - name: Delete mysql users.
   community.mysql.mysql_user:
     name: "{{ database.user }}_{{ item }}"
@@ -24,6 +23,6 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   when: mysql_backup.credentials_handling == 'rotate'
   run_once: true
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index a5da0d87..6f8dc1f6 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -2,14 +2,14 @@
 - name: Ensure codebase is writable.
   ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi"
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   become: true
   when: "www_user != deploy_user"
 
 - name: Ensure permissions are set on deploy directory.
   ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
     - deploy_code.perms_fix_path is defined
     - deploy_code.perms_fix_path | length > 1
@@ -18,7 +18,7 @@
 - name: Ensure permissions are set on builds directory.
   ansible.builtin.shell:
     cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
     - deploy_code.perms_fix_path is defined
     - deploy_code.perms_fix_path | length > 1
@@ -28,7 +28,7 @@
   ansible.builtin.file:
     name: "{{ deploy_path_prefix }}{{ item }}"
     state: absent
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
     - deploy_code.mount_type != "squashfs"
 
@@ -36,7 +36,7 @@
   ansible.builtin.file:
     name: "{{ build_path_prefix }}{{ item }}"
     state: absent
-  with_sequence: start={{ [previous_build_number | int - 50, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   when:
     - deploy_code.mount_type == "squashfs"
 

From dd1f5cea2f1dd646999e17f607a863d3ee133547 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 12 Apr 2023 10:26:47 +0200
Subject: [PATCH 053/139] Adding escaped backticks to db names to be safe.
 (#256)

---
 .../database_backup-mysql/tasks/deploy-rolling.yml   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 71fa7580..88b51523 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -7,10 +7,20 @@
   # Note: we don't use the mysql_db Ansible module on purpose.
   # If database already exists, we want to fail and not override it
   # with previous build.
+  # @TODO fix this so we check if the database exists and exit with
+  # the proper plugin instead of using command.
 - name: Create new database.
-  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE \`{{ _mysql_build_database_name }}\`;"
   run_once: true
 
+#- name: Create a new database.
+#  community.mysql.mysql_db:
+#    name: "{{ _mysql_build_database_name }}"
+#    state: present
+#    config_file: "{{ database.credentials_file }}"
+#    config_overrides_defaults: true
+#  run_once: true
+
 - name: Populate new database.
   ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
   when: previous_build_number > 0

From df74ac6032a9327da6e0d6c158adf7dd64e2e3cc Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 12 Apr 2023 10:37:47 +0200
Subject: [PATCH 054/139] Mysql db name pr 1.x (#258)

* Adding escaped backticks to db names to be safe.

* Looks like Ansible auto-escapes backticks.
---
 .../database_backup-mysql/tasks/deploy-rolling.yml              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 88b51523..f7f109bc 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -10,7 +10,7 @@
   # @TODO fix this so we check if the database exists and exit with
   # the proper plugin instead of using command.
 - name: Create new database.
-  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE \`{{ _mysql_build_database_name }}\`;"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE `{{ _mysql_build_database_name }}`;"
   run_once: true
 
 #- name: Create a new database.

From 3b49dc9775c41abf00ddf6013894a9e27cd46b51 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 12 Apr 2023 10:44:22 +0200
Subject: [PATCH 055/139] Mysql db name pr 1.x (#260)

* Adding escaped backticks to db names to be safe.

* Looks like Ansible auto-escapes backticks.

* Adding backticks to database names for creating MySQL users too.
---
 roles/database_backup/database_backup-mysql/tasks/deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index 5442230d..4160806c 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -90,12 +90,12 @@
 # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL.
 # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
 - name: Create/update mysql user for TCP connections.
-  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'%';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 
 - name: Create/update mysql user for unix socket connections.
-  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';"
+  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'localhost';"
   when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
   run_once: true
 

From 8c30320c1871fd031c4e98555771c6f7ae143a12 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 13 Apr 2023 17:22:37 +0200
Subject: [PATCH 056/139] Mysql db name pr 1.x (#262)

* Adding escaped backticks to db names to be safe.

* Looks like Ansible auto-escapes backticks.

* Adding backticks to database names for creating MySQL users too.

* Adding extra mysqldump flags to stop restores failing on RDS.
---
 roles/_init/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 971f1b9b..5eed81f2 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -3,7 +3,7 @@
 # this will ensure defaults to other roles too.
 # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
-_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+_mysqldump_params: "--set-gtid-purged=OFF --skip-definer --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"

From 15ae9015a5712b1cf9f0512672afcc28430304a2 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 14 Apr 2023 12:49:16 +0200
Subject: [PATCH 057/139] Mysql db name pr 1.x (#264)

* Adding escaped backticks to db names to be safe.

* Looks like Ansible auto-escapes backticks.

* Adding backticks to database names for creating MySQL users too.

* Adding extra mysqldump flags to stop restores failing on RDS.

* Removing ignore_errors.

* Updating mysqldump query.
---
 roles/_init/defaults/main.yml       | 3 ++-
 roles/deploy_code/tasks/cleanup.yml | 2 +-
 roles/lhci_run/tasks/main.yml       | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 5eed81f2..2602afd2 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -3,7 +3,8 @@
 # this will ensure defaults to other roles too.
 # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
-_mysqldump_params: "--set-gtid-purged=OFF --skip-definer --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
+_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
     - folder: "default"
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 6f8dc1f6..27878342 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -128,7 +128,7 @@
 - name: Check if we have a mount already.
   ansible.builtin.shell:
     cmd: "mount | grep {{ deploy_base_path }}"
-  ignore_errors: true
+  failed_when: false
   register: _deploy_code_mount_check
   when:
     - deploy_code.mount_sync is defined
diff --git a/roles/lhci_run/tasks/main.yml b/roles/lhci_run/tasks/main.yml
index 6e9648ce..e5a596fe 100644
--- a/roles/lhci_run/tasks/main.yml
+++ b/roles/lhci_run/tasks/main.yml
@@ -2,12 +2,12 @@
 - name: Check if 'lhci' is available.
   ansible.builtin.command: "which lhci"
   register: _lhci_run_check_result
-  ignore_errors: true
+  failed_when: false
 
 - name: Check if 'Xvfb' is running.
   ansible.builtin.shell: "pgrep Xvfb"
   register: _lhci_run_check_xvfb_result
-  ignore_errors: true
+  failed_when: false
   when:
     - _lhci_run_check_result.rc == 0
 

From c7e4a472e96706c39d38d62b24d1ab5a5d895df6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 14 Apr 2023 20:35:03 +0200
Subject: [PATCH 058/139] Bug fixes pr 1.x (#266)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.
---
 .ansible-lint                                      | 11 ++++++++---
 .github/workflows/ce-deploy-test.yml               |  2 +-
 .../cache_clear/cache_clear-opcache/tasks/main.yml | 10 ++++++----
 roles/cli/cachetool/tasks/main.yml                 | 11 ++++++-----
 roles/cli/drush/tasks/main.yml                     |  4 +++-
 .../database_apply-drupal7/tasks/ctools.yml        |  4 +++-
 .../database_apply-drupal7/tasks/features.yml      |  4 +++-
 .../database_apply-drupal7/tasks/main.yml          |  6 ++++--
 .../database_apply-drupal8/tasks/main.yml          | 14 ++++++++------
 .../database_backup-mysql/tasks/deploy-dump.yml    |  4 +++-
 .../database_backup-mysql/tasks/deploy-rolling.yml |  4 +++-
 .../database_backup-mysql/tasks/revert-dump.yml    |  4 +++-
 roles/deploy_code/tasks/cleanup.yml                |  4 +++-
 .../database_sync-mysql/tasks/sync.yml             |  8 ++++++--
 14 files changed, 60 insertions(+), 30 deletions(-)

diff --git a/.ansible-lint b/.ansible-lint
index 7a9eb05b..b6256e0b 100644
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -1,4 +1,9 @@
 skip_list:
-  - '204'
-  - '301'
-  - '701'
+  - '204' # Lines should be no longer than 160 chars
+  - '301' # Commands should not change things if nothing needs doing
+  - '701' # 701 No 'galaxy_info' found
+  - unnamed-task
+  - risky-file-permissions
+  - no-jinja-nesting
+  - command-instead-of-shell # some Drupal 7 drush commands require shell
+  - command-instead-of-module
diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml
index 141fc8e7..c6429842 100644
--- a/.github/workflows/ce-deploy-test.yml
+++ b/.github/workflows/ce-deploy-test.yml
@@ -10,7 +10,7 @@ jobs:
     # Name the Job
     name: Run tests against Ansible code base
     # Set the type of machine to run on
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       # Checks out a copy of your repository on the ubuntu-latest machine
diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
index 428e9053..97f91ed4 100644
--- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml
@@ -1,7 +1,9 @@
 ---
 - name: Get latest php installed
   ansible.builtin.shell:
-    cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1'
+    cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1'
+  args:
+    executable: /bin/bash
   register: _php_version
 
 - name: Set cachetool adapter.
@@ -9,16 +11,16 @@
     _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('--fcgi=127.0.0.1:90' + _php_version.stdout | replace('.','')) }}"
 
 - name: Clear opcache.
-  command:
+  ansible.builtin.command:
     cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset"
   when: cache_clear_opcache.clear_opcache
 
 - name: Clear apcu.
-  command:
+  ansible.builtin.command:
     cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear"
   when: cache_clear_opcache.clear_apcu
 
 - name: Clear stats.
-  command:
+  ansible.builtin.command:
     cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear"
   when: cache_clear_opcache.clear_stat
diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index 2e2a49d1..c0cbce61 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -14,10 +14,15 @@
     - deploy_operation == 'deploy'
 
 - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility
+  when:
+    - deploy_operation == 'deploy'
+    - cachetool.version | length == 0
   block:
     - name: Get latest php installed
       ansible.builtin.shell:
-        cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"'
+        cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"'
+      args:
+        executable: /bin/bash
       register: _php_version_str
 
     - name: Download latest cachetool installer if PHP is 8.1 or newer.
@@ -69,10 +74,6 @@
       when:
         - _php_version_str.stdout | int < 71
 
-  when:
-    - deploy_operation == 'deploy'
-    - cachetool.version | length == 0
-
 - name: "Download the specified {{ cachetool.version }} cachetool version installer."
   ansible.builtin.get_url:
     url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml
index 5c73be51..0836c91f 100644
--- a/roles/cli/drush/tasks/main.yml
+++ b/roles/cli/drush/tasks/main.yml
@@ -16,7 +16,9 @@
   register: drush_global_directory
 
 - name: Check if installed Drush version match.
-  ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'"
+  ansible.builtin.shell: "set -o pipefail && {{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'"
+  args:
+    executable: /bin/bash
   register: drush_global_version
   when:
     - deploy_operation == 'deploy'
diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
index 8102d3d6..518996ab 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
@@ -2,8 +2,10 @@
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Check if Ctools module is enabled.
   ansible.builtin.shell:
-    cmd: "{{ drush_bin }} pm-info ctools | grep ':  enabled' | wc -l"
+    cmd: "set -o pipefail && {{ drush_bin }} pm-info ctools | grep ':  enabled' | wc -l"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
+    executable: /bin/bash
   register: ctools_enabled
 
 - name: Revert Drupal configuration from Ctools.
diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml
index f55795dc..2b4357eb 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/features.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml
@@ -1,7 +1,9 @@
 ---
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Check if Features module is enabled.
-  ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ':  enabled' | wc -l"
+  ansible.builtin.shell: "set -o pipefail && cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ':  enabled' | wc -l"
+  args:
+    executable: /bin/bash
   register: features_enabled
 
 - name: Revert Drupal configuration from Features.
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 6b745c3b..d64364c4 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -12,8 +12,10 @@
   when: previous_build_number == 0 or (site.force_install is defined and site.force_install)
 
 - name: Fix permissions on Drupal directory.
-  ansible.builtin.shell:
-    cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  ansible.builtin.file:
+    path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    state: directory
+    mode: '0755'
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 16f0417b..4dc66a3a 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Fix file permissions for settings.php.
-  file:
+  ansible.builtin.file:
     state: file
     path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php"
     owner: "{{ www_user }}"
@@ -14,7 +14,7 @@
     - previous_build_number == 0
 
 - name: Install Drupal.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
@@ -25,8 +25,10 @@
   when: (previous_build_number == 0) or (site.force_install is defined and site.force_install)
 
 - name: Fix permissions on Drupal directory.
-  shell:
-    cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  ansible.builtin.file:
+    path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    state: directory
+    mode: '0755'
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
@@ -48,7 +50,7 @@
   when: previous_build_number > 0
 
 - name: Apply Drupal database updates.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
@@ -59,7 +61,7 @@
   when: site.config_import_command != 'deploy'
 
 - name: Import configuration.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 80a6e5ef..9f6f6959 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -23,6 +23,8 @@
   run_once: true
 
 - name: Take a database dump.
-  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  args:
+    executable: /bin/bash
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index f7f109bc..aec45596 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -22,6 +22,8 @@
 #  run_once: true
 
 - name: Populate new database.
-  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
+  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
+  args:
+    executable: /bin/bash
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index f914ee78..80898e7d 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -1,5 +1,7 @@
 ---
 - name: Revert database from dump.
-  ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}"
+  ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}"
+  args:
+    executable: /bin/bash
   when: previous_build_number > 0
   run_once: true
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 27878342..db2bb9bb 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -127,7 +127,9 @@
 
 - name: Check if we have a mount already.
   ansible.builtin.shell:
-    cmd: "mount | grep {{ deploy_base_path }}"
+    cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}"
+  args:
+    executable: /bin/bash
   failed_when: false
   register: _deploy_code_mount_check
   when:
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 29100ab5..c695982c 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -61,7 +61,9 @@
   when: not database.source.type == 'rolling'
 
 - name: Take a dump from source database.
-  ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  args:
+    executable: /bin/bash
   delegate_to: "{{ database.source.host }}"
   when:
     - database.source.fresh_db
@@ -133,7 +135,9 @@
     cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
 
 - name: Repopulate database from dump.
-  ansible.builtin.shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
+  ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
+  args:
+    executable: /bin/bash
 
 - name: Remove tmp dump file.
   ansible.builtin.file:

From 5b28bdd1ea6634d1b3336c9e7135a80ef3d23689 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 19 Apr 2023 19:30:27 +0200
Subject: [PATCH 059/139] Bug fixes pr 1.x (#268)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.
---
 docs/roles/_init.md                                          | 5 +++++
 roles/_init/README.md                                        | 5 +++++
 roles/_init/defaults/main.yml                                | 2 +-
 roles/database_apply/database_apply-drupal7/tasks/ctools.yml | 2 +-
 roles/database_apply/database_apply-drupal7/tasks/main.yml   | 3 ++-
 5 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 1c7e835e..9cb9e024 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 # this will ensure defaults to other roles too.
 # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
+# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
@@ -23,6 +24,7 @@ drupal:
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
+      revert_ctools_command: "ctools-export-revert --all"
       # End Drupal 7 variables
       sanitize_command: "sql-sanitize"
       base_url: https://www.example.com
@@ -35,6 +37,9 @@ mautic:
   image_path: "media/images"
   force_install: false
 bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 1c7e835e..9cb9e024 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 # this will ensure defaults to other roles too.
 # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
 deploy_user: "deploy"
+# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   sites:
@@ -23,6 +24,7 @@ drupal:
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
+      revert_ctools_command: "ctools-export-revert --all"
       # End Drupal 7 variables
       sanitize_command: "sql-sanitize"
       base_url: https://www.example.com
@@ -35,6 +37,9 @@ mautic:
   image_path: "media/images"
   force_install: false
 bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 2602afd2..9a955f3e 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -15,7 +15,7 @@ drupal:
       # End Drupal 8 variables
       # Drupal 7 variables
       revert_features_command: "" # i.e. "fra"
-      revert_ctools: true
+      revert_ctools_command: "ctools-export-revert --all"
       # End Drupal 7 variables
       sanitize_command: "sql-sanitize"
       base_url: https://www.example.com
diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
index 518996ab..608e1a2f 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml
@@ -10,6 +10,6 @@
 
 - name: Revert Drupal configuration from Ctools.
   ansible.builtin.shell:
-    cmd: "{{ drush_bin }} -y ctools-export-revert --all"
+    cmd: "{{ drush_bin }} -y {{ site.revert_ctools_command }}"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   when: ctools_enabled.stdout == "1"
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index d64364c4..4f476253 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -54,4 +54,5 @@
   loop_control:
     loop_var: site
   when:
-    - site.revert_ctools
+    - previous_build_number > 0
+    - site.revert_ctools_command | length > 0

From ce46e32fe437daf6680beee5255d58df148e09ed Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 20 Apr 2023 08:25:20 +0200
Subject: [PATCH 060/139] Making sync paths build unique.

---
 .../database_sync/database_sync-mysql/tasks/sync.yml   |  4 ++--
 roles/sync/files_sync/tasks/sync.yml                   | 10 +++++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index c695982c..9b949f27 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -37,7 +37,7 @@
 
 - name: Register remote dump name (from database).
   ansible.builtin.set_fact:
-    mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2"
+    mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.bz2"
 
 - name: Get source last known good build number.
   ansible.builtin.command:
@@ -91,7 +91,7 @@
 
 - name: Register tmp target dump name.
   ansible.builtin.set_fact:
-    mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2"
+    mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2"
 
 - name: Get target last known good build number.
   ansible.builtin.command:
diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml
index e2160008..58ca4cd0 100644
--- a/roles/sync/files_sync/tasks/sync.yml
+++ b/roles/sync/files_sync/tasks/sync.yml
@@ -1,7 +1,11 @@
 ---
+- name: Register file sync location.
+  ansible.builtin.set_fact:
+    file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}"
+
 - name: Create a temporary directory for source files on localhost.
   ansible.builtin.file:
-    path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}"
+    path: "{{ file_sync_path }}"
     state: directory
     owner: "{{ deploy_user }}"
     group: "{{ deploy_user }}"
@@ -10,12 +14,12 @@
 
 - name: Copy the source files onto the deploy server.
   ansible.builtin.command:
-    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/"
+    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/"
   delegate_to: "localhost"
   run_once: true
 
 - name: Copy the source files from the deploy server onto the destination server.
   ansible.builtin.command:
-    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/"
+    cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/"
   delegate_to: "localhost"
   run_once: true

From aa9d3b0329852029f4a1ecbbfdac34fc40dbb6a3 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 21 Apr 2023 17:47:19 +0200
Subject: [PATCH 061/139] Adding --host parameter to build.sh for pre-deploy
 host checking with Ansible. (#271)

---
 scripts/_common.sh     | 24 ++++++++++++++++++++++++
 scripts/build.sh       |  4 ++++
 scripts/host-check.yml | 15 +++++++++++++++
 3 files changed, 43 insertions(+)
 create mode 100644 scripts/host-check.yml

diff --git a/scripts/_common.sh b/scripts/_common.sh
index 3715c97a..63e89bb9 100755
--- a/scripts/_common.sh
+++ b/scripts/_common.sh
@@ -9,6 +9,7 @@ export ANSIBLE_CONFIG="$OWN_DIR/ansible.cfg"
 TARGET_DEPLOY_REPO=""
 TARGET_DEPLOY_PLAYBOOK=""
 TARGET_DEPLOY_BRANCH=""
+TARGET_DEPLOY_HOST=""
 PREVIOUS_BUILD_NUMBER=""
 CURRENT_BUILD_NUMBER=""
 ANSIBLE_EXTRA_VARS=""
@@ -49,6 +50,10 @@ parse_options(){
           shift
           TARGET_DEPLOY_PLAYBOOK="$1"
         ;;
+      "--host")
+          shift
+          TARGET_DEPLOY_HOST="$1"
+        ;;
       "--build-number")
           shift
           CURRENT_BUILD_NUMBER="$1"
@@ -146,6 +151,25 @@ cleanup_build_tmp_dir(){
   fi
 }
 
+# Call Ansible playbook to ensure host exists.
+ansible_host_check(){
+  if [ -n "$TARGET_DEPLOY_HOST" ]; then
+    ANSIBLE_BIN=$(command -v ansible-playbook)
+    ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml"
+    if [ "$DRY_RUN" = "yes" ]; then
+      ANSIBLE_CMD="$ANSIBLE_CMD --check"
+    fi
+    if [ "$VERBOSE" = "yes" ]; then
+      ANSIBLE_CMD="$ANSIBLE_CMD -vvvv"
+    fi
+    if [ -n "$BOTO_PROFILE" ]; then
+      export AWS_PROFILE="$BOTO_PROFILE"
+    fi
+    $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS"
+    return $?
+  fi
+}
+
 # Trigger actual Ansible job.
 # $1 (string)
 # Operation to perform.
diff --git a/scripts/build.sh b/scripts/build.sh
index 5f7fa21b..233ca764 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -19,6 +19,7 @@ usage(){
   echo '--branch: The branch to deploy.'
   echo ''
   echo 'Available options:'
+  echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.'
   echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.'
   echo '--previous-stable-build-number: an incremental build number that '
   echo '--dry-run: Do not perform any action but run the playbooks in --check mode.'
@@ -94,6 +95,9 @@ fi
 # Get Ansible defaults.
 get_ansible_defaults_vars
 
+# Optionally carry out a host check.
+ansible_host_check
+
 # From this point on, we want to trigger the "revert" if anything fails.
 ANSIBLE_BUILD_RESULT=1
 # Trigger deploy.
diff --git a/scripts/host-check.yml b/scripts/host-check.yml
new file mode 100644
index 00000000..980c63cb
--- /dev/null
+++ b/scripts/host-check.yml
@@ -0,0 +1,15 @@
+- hosts: localhost
+  connection: local
+  become: false
+  tasks:
+    - name: Ensure the hostname check variable is empty.
+      ansible.builtin.set_fact:
+        _ce_deploy_ansible_host_check: ""
+    - name: Check to see if the Ansible host or hostgroup exists.
+      ansible.builtin.set_fact:
+        _ce_deploy_ansible_host_check: "{{ item }}"
+      with_inventory_hostnames:
+        - "{{ _deploy_host }}"
+    - ansible.builtin.fail:
+        msg: "Host does not exist!"
+      when: _ce_deploy_ansible_host_check | length == 0

From fd7c17d4a002f3e9cec161bea9c0e6d992f55a9f Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 25 Apr 2023 10:04:31 +0200
Subject: [PATCH 062/139] Making syncs safer and more efficient. (#273)

---
 .../database_sync-mysql/defaults/main.yml     |  1 +
 .../database_sync-mysql/tasks/sync.yml        | 20 ++++++++++++++++---
 roles/sync/files_sync/defaults/main.yml       |  3 +++
 roles/sync/files_sync/tasks/sync.yml          | 17 ++++++++++++++--
 4 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 0ab99e98..48c02b8b 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -1,6 +1,7 @@
 ---
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 9b949f27..7815868f 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -104,12 +104,12 @@
   delegate_to: localhost
   when: database.target.type == 'rolling'
 
-- name: Register target database name.
+- name: Register target rolling database name.
   ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}"
   when: database.target.type == 'rolling'
 
-- name: Register target database name.
+- name: Register target static database name.
   ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}"
   when: not database.target.type == 'rolling'
@@ -139,11 +139,25 @@
   args:
     executable: /bin/bash
 
-- name: Remove tmp dump file.
+- name: Delete temporary dump file on target.
   ansible.builtin.file:
     path: "{{ mysql_sync_target_dump_path }}"
     state: absent
 
+- name: Delete temporary dump file on source.
+  ansible.builtin.file:
+    path: "{{ mysql_sync_source_dump_path }}"
+    state: absent
+  delegate_to: "{{ database.source.host }}"
+
+- name: Delete temporary dump file on deploy server.
+  ansible.builtin.file:
+    path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
+    state: absent
+  delegate_to: localhost
+  when:
+    - mysql_sync.cleanup
+
 - name: Enable all autoscale processes on source ASG.
   ansible.builtin.command: >
     aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml
index 5b65b1fb..4e222b40 100644
--- a/roles/sync/files_sync/defaults/main.yml
+++ b/roles/sync/files_sync/defaults/main.yml
@@ -1,5 +1,8 @@
 ---
 files_sync:
+  unique_workspace: false # set to true to grab a complete full set of files every sync
+  # Generally speaking you will *not* want to clean up after file syncs, as leaving the files there makes the next rsync far quicker.
+  cleanup: false # set to true to delete the synced files after a sync
   directories:
     - source:
         # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH!
diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml
index 58ca4cd0..516059c2 100644
--- a/roles/sync/files_sync/tasks/sync.yml
+++ b/roles/sync/files_sync/tasks/sync.yml
@@ -1,7 +1,12 @@
 ---
 - name: Register file sync location.
+  ansible.builtin.set_fact:
+    file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}"
+
+- name: Register unique file sync location.
   ansible.builtin.set_fact:
     file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}"
+  when: files_sync.unique_workspace
 
 - name: Create a temporary directory for source files on localhost.
   ansible.builtin.file:
@@ -15,11 +20,19 @@
 - name: Copy the source files onto the deploy server.
   ansible.builtin.command:
     cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/"
-  delegate_to: "localhost"
+  delegate_to: localhost
   run_once: true
 
 - name: Copy the source files from the deploy server onto the destination server.
   ansible.builtin.command:
     cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/"
-  delegate_to: "localhost"
+  delegate_to: localhost
   run_once: true
+
+- name: Delete synced files on deploy server.
+  ansible.builtin.file:
+    path: "{{ file_sync_path }}"
+    state: absent
+  delegate_to: localhost
+  when:
+    - files_sync.cleanup

From ac8b1144dee9189b6830e4f901542f54d3ca02a3 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 11 May 2023 14:09:25 +0200
Subject: [PATCH 063/139] Bug fixes pr 1.x (#278)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.
---
 docs/roles/sync/database_sync/database_sync-mysql.md   |  1 +
 .../config_generate-drupal7/tasks/settings.yml         |  2 +-
 roles/deploy_code/deploy_code-custom/tasks/main.yml    |  3 +++
 roles/sync/database_sync/database_sync-mysql/README.md |  1 +
 scripts/_common.sh                                     |  6 +++---
 scripts/build.sh                                       | 10 ++++++++--
 6 files changed, 17 insertions(+), 6 deletions(-)
 create mode 100644 roles/deploy_code/deploy_code-custom/tasks/main.yml

diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index cf7e5311..cbe1a353 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -6,6 +6,7 @@ Sync MySQL databases between environments.
 ---
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
index f8e3b55e..55d9dee7 100644
--- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml
@@ -19,6 +19,6 @@
     src: "{{ item }}"
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php"
   with_first_found:
-    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2"
+    - "{{ playbook_dir }}/{{ site.folder }}/{{ build_type }}.settings.php.j2"
     - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php"
     - "settings.php.j2"
diff --git a/roles/deploy_code/deploy_code-custom/tasks/main.yml b/roles/deploy_code/deploy_code-custom/tasks/main.yml
new file mode 100644
index 00000000..6f34c92f
--- /dev/null
+++ b/roles/deploy_code/deploy_code-custom/tasks/main.yml
@@ -0,0 +1,3 @@
+---
+
+# Nothing to do here.
\ No newline at end of file
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index cf7e5311..cbe1a353 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -6,6 +6,7 @@ Sync MySQL databases between environments.
 ---
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/scripts/_common.sh b/scripts/_common.sh
index 63e89bb9..74ba9a60 100755
--- a/scripts/_common.sh
+++ b/scripts/_common.sh
@@ -156,9 +156,6 @@ ansible_host_check(){
   if [ -n "$TARGET_DEPLOY_HOST" ]; then
     ANSIBLE_BIN=$(command -v ansible-playbook)
     ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml"
-    if [ "$DRY_RUN" = "yes" ]; then
-      ANSIBLE_CMD="$ANSIBLE_CMD --check"
-    fi
     if [ "$VERBOSE" = "yes" ]; then
       ANSIBLE_CMD="$ANSIBLE_CMD -vvvv"
     fi
@@ -167,6 +164,9 @@ ansible_host_check(){
     fi
     $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS"
     return $?
+  # No host to check provided, just return a clean exit code.
+  else
+    return 0
   fi
 }
 
diff --git a/scripts/build.sh b/scripts/build.sh
index 233ca764..93f36887 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -95,8 +95,14 @@ fi
 # Get Ansible defaults.
 get_ansible_defaults_vars
 
-# Optionally carry out a host check.
+# Optionally carry out a host check if --host is provided.
 ansible_host_check
+ANSIBLE_HOST_CHECK_RESULT=$?
+# Exit early if host not found.
+if [ -n "$ANSIBLE_HOST_CHECK_RESULT" ] && [ "$ANSIBLE_HOST_CHECK_RESULT" != 0 ]; then
+  echo "ce-deploy failed to find the host. Aborting."
+  exit 1
+fi
 
 # From this point on, we want to trigger the "revert" if anything fails.
 ANSIBLE_BUILD_RESULT=1
@@ -113,5 +119,5 @@ if [ -n "$ANSIBLE_BUILD_RESULT" ] && [ "$ANSIBLE_BUILD_RESULT" = 0 ]; then
   exit 0
 fi
 # Failed somehow. Normally unreachable in strict mode.
-echo "Something went wrong. Please fill a bug report against ce-deploy."
+echo "Something went unexpectedly wrong with ce-deploy. Please file a bug report - https://github.com/codeenigma/ce-deploy/issues/new"
 exit 1
\ No newline at end of file

From 430b731f422f9c9e486de9939aeeb190533c62be Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 12 May 2023 14:51:19 +0200
Subject: [PATCH 064/139] Bug fixes pr 1.x (#282)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.
---
 docs/roles/_init.md           | 1 +
 roles/_init/README.md         | 1 +
 roles/_init/defaults/main.yml | 1 +
 roles/_init/tasks/main.yml    | 1 +
 4 files changed, 4 insertions(+)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 9cb9e024..08ffe301 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -39,6 +39,7 @@ mautic:
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
 
 ```
 
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 9cb9e024..08ffe301 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -39,6 +39,7 @@ mautic:
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
 
 ```
 
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 9a955f3e..ac06e2a7 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -30,3 +30,4 @@ mautic:
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index a0d90571..c054e8fb 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -124,3 +124,4 @@
 - name: Ensure we have a cachetool binary.
   ansible.builtin.import_role:
     name: cli/cachetool
+  when: install_php_cachetool

From 4d211b5a600505c86f3c803a9bf6baec5493b325 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 17 May 2023 15:47:49 +0200
Subject: [PATCH 065/139] Bug fixes pr 1.x (#284)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.
---
 roles/deploy_code/tasks/cleanup.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index db2bb9bb..b3a4f132 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -150,7 +150,7 @@
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check is succeeded
+    - _deploy_code_mount_check.rc == 0
     - deploy_code.services | length > 0
 
 - name: Stop any services that might be keeping the loop device busy.
@@ -166,7 +166,7 @@
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check is succeeded
+    - _deploy_code_mount_check.rc == 0
     - deploy_code.services | length > 0
 
 - name: Unmount existing SquashFS image.
@@ -177,7 +177,7 @@
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check is succeeded
+    - _deploy_code_mount_check.rc == 0
 
 - name: Mount new SquashFS image.
   ansible.builtin.command:

From 7dbc30caba2ce5bfa6c2719b151d6714137bba46 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 25 May 2023 15:34:43 +0200
Subject: [PATCH 066/139] Bug fixes pr 1.x (#287)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.
---
 roles/_meta/deploy-drupal8/tasks/main.yml                  | 4 ++--
 roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 -----
 roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 -----
 3 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml
index 6ce31d4f..3e66875b 100644
--- a/roles/_meta/deploy-drupal8/tasks/main.yml
+++ b/roles/_meta/deploy-drupal8/tasks/main.yml
@@ -20,14 +20,14 @@
     name: database_backup
 - import_role:
     name: config_generate
+- import_role:
+    name: cache_clear/cache_clear-opcache
 - import_role:
     name: database_apply
 - import_role:
     name: sanitize/admin_creds
 - import_role:
     name: live_symlink
-- import_role:
-    name: cache_clear/cache_clear-opcache
 - import_role:
     name: cache_clear/cache_clear-drupal8
 - import_role:
diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml
index 4f476253..3012032f 100644
--- a/roles/database_apply/database_apply-drupal7/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml
@@ -26,11 +26,6 @@
     name: "cache_clear/cache_clear-{{ project_type }}"
   when: previous_build_number > 0
 
-- name: Clear the opcache.
-  ansible.builtin.include_role:
-    name: cache_clear/cache_clear-opcache
-  when: previous_build_number > 0
-
 - name: Apply Drupal database updates.
   ansible.builtin.shell:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 4dc66a3a..c649826c 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -44,11 +44,6 @@
     - previous_build_number > 0
     - site.config_import_command != 'deploy'
 
-- name: Clear the opcache.
-  ansible.builtin.include_role:
-    name: cache_clear/cache_clear-opcache
-  when: previous_build_number > 0
-
 - name: Apply Drupal database updates.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"

From 1e540b4ed34fb744ee1620af1012dc54fd5af901 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 21 Jun 2023 12:34:46 +0200
Subject: [PATCH 067/139] Adding verbose output for drush. (#289)

---
 docs/roles/_init.md                           |  1 +
 roles/_init/README.md                         |  1 +
 roles/_init/defaults/main.yml                 |  1 +
 .../cache_clear-drupal8/tasks/main.yml        |  9 ++++++++-
 .../database_apply-drupal8/tasks/main.yml     | 19 +++++++++++++++++++
 .../database_apply-drupal8/tasks/main.yml     | 12 ++++++++++++
 6 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 08ffe301..edf1f5d3 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -15,6 +15,7 @@ deploy_user: "deploy"
 # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
+  drush_verbose_output: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 08ffe301..edf1f5d3 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -15,6 +15,7 @@ deploy_user: "deploy"
 # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
+  drush_verbose_output: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index ac06e2a7..57ee7e44 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -6,6 +6,7 @@ deploy_user: "deploy"
 # for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
+  drush_verbose_output: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index 5b780086..e9524b81 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,6 +1,6 @@
 ---
 - name: Clear Drupal cache.
-  command:
+  ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
     chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
@@ -9,3 +9,10 @@
   loop_control:
     loop_var: site
   run_once: true
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
+
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index c649826c..e2fecd17 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -23,6 +23,12 @@
   loop_control:
     loop_var: site
   when: (previous_build_number == 0) or (site.force_install is defined and site.force_install)
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
 
 - name: Fix permissions on Drupal directory.
   ansible.builtin.file:
@@ -54,6 +60,12 @@
   loop_control:
     loop_var: site
   when: site.config_import_command != 'deploy'
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
 
 - name: Import configuration.
   ansible.builtin.command:
@@ -67,3 +79,10 @@
   when:
     - previous_build_number > 0
     - site.config_import_command
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
+
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
index 96d9b65f..d13b0129 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
@@ -17,6 +17,12 @@
   with_items: "{{ drupal.sites }}"
   loop_control:
     loop_var: site
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
 
 # This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run.
 - name: Import configuration.
@@ -31,6 +37,12 @@
   when:
     - site.config_import_command
     - site.sync_config_import is not defined or site.sync_config_import
+  register: _drush_output
+
+- name: Show drush output.
+  ansible.builtin.debug:
+    msg: "{{ _drush_output }}"
+  when: drupal.drush_verbose_output
 
 - name: Clear the cache.
   ansible.builtin.include_role:

From 008a1e0823c89360f32995667ce546164eda5e7c Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 22 Jun 2023 19:02:02 +0200
Subject: [PATCH 068/139] Adding container push and build code. (#300)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.
---
 docs/_Sidebar.md                            |  1 +
 docs/roles/deploy_container.md              | 38 +++++++++++++++
 roles/deploy_container/README.md            | 38 +++++++++++++++
 roles/deploy_container/defaults/main.yml    | 16 +++++++
 roles/deploy_container/tasks/main.yml       | 51 +++++++++++++++++++++
 roles/deploy_container/templates/example.j2 |  7 +++
 6 files changed, 151 insertions(+)
 create mode 100644 docs/roles/deploy_container.md
 create mode 100644 roles/deploy_container/README.md
 create mode 100644 roles/deploy_container/defaults/main.yml
 create mode 100644 roles/deploy_container/tasks/main.yml
 create mode 100644 roles/deploy_container/templates/example.j2

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index 16139d8f..3d47469a 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -31,6 +31,7 @@
      - [Data backups](/roles/database_backup)
        - [MySQL backups](/roles/database_backup/database_backup-mysql)
      - [Deploy](/roles/deploy_code)
+     - [Deploy container](/roles/deploy_container)
      - [Init](/roles/_init)
      - [LHCI run](/roles/lhci_run)
      - ["Meta"](/roles/_meta)
diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
new file mode 100644
index 00000000..9730b01e
--- /dev/null
+++ b/docs/roles/deploy_container.md
@@ -0,0 +1,38 @@
+# Deploy container
+Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example:
+
+```
+sudo groupadd docker
+sudo usermod -aG docker deploy
+```
+
+This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
+
+AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+deploy_container:
+  container_name: example
+  container_tag: latest # tag will take format container_name:container_tag
+  docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
+  docker_registry_user: example
+  docker_registry_pass: asdf1234
+  docker_base_command: "docker image build"
+  docker_build_dir: "{{ _ce_deploy_build_dir }}"
+  dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
+  aws_ecr:
+    enabled: false # set to true if using AWS ECR
+    region: eu-west-1
+    profile: example
+
+```
+
+<!--ENDROLEVARS-->
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
new file mode 100644
index 00000000..9730b01e
--- /dev/null
+++ b/roles/deploy_container/README.md
@@ -0,0 +1,38 @@
+# Deploy container
+Step that deploys the codebase in a Docker container image. Requires Docker and the `community.docker` collection for Ansible to be installed on your deploy server. You will also need to add a `docker` group and make sure your local deploy user is in that group, for example:
+
+```
+sudo groupadd docker
+sudo usermod -aG docker deploy
+```
+
+This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
+
+AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+deploy_container:
+  container_name: example
+  container_tag: latest # tag will take format container_name:container_tag
+  docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
+  docker_registry_user: example
+  docker_registry_pass: asdf1234
+  docker_base_command: "docker image build"
+  docker_build_dir: "{{ _ce_deploy_build_dir }}"
+  dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
+  aws_ecr:
+    enabled: false # set to true if using AWS ECR
+    region: eu-west-1
+    profile: example
+
+```
+
+<!--ENDROLEVARS-->
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
new file mode 100644
index 00000000..3578e12a
--- /dev/null
+++ b/roles/deploy_container/defaults/main.yml
@@ -0,0 +1,16 @@
+---
+deploy_container:
+  container_name: example
+  container_tag: latest # tag will take format container_name:container_tag
+  docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
+  docker_registry_user: example
+  docker_registry_pass: asdf1234
+  docker_base_command: "docker image build"
+  docker_build_dir: "{{ _ce_deploy_build_dir }}"
+  dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
+  aws_ecr:
+    enabled: false # set to true if using AWS ECR
+    region: eu-west-1
+    profile: example
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
new file mode 100644
index 00000000..c58bb8b1
--- /dev/null
+++ b/roles/deploy_container/tasks/main.yml
@@ -0,0 +1,51 @@
+---
+# @TODO - for AWS ECR we'll need certain policies attaching to the deploy IAM user
+- name: Create Dockerfile from template.
+  local_action:
+    module: ansible.builtin.template
+    src: "{{ deploy_container.dockerfile_template }}"
+    dest: "{{ deploy_container.docker_build_dir }}/Dockerfile"
+
+- name: Set Docker registry username and password.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "{{ deploy_container.docker_registry_user }}"
+    _docker_registry_password: "{{ deploy_container.docker_registry_pass }}"
+  delegate_to: localhost
+
+# Token valid for 12 hours
+- name: Fetch AWS ECR registry login token.
+  ansible.builtin.command:
+    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.profile }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+  register: _docker_registry_ecr_token
+
+- name: Set AWS ECR registry password.
+  ansible.builtin.set_fact:
+    _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Set AWS ECR registry username.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "AWS"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Log into Docker registry.
+  community.docker.docker_login:
+    registry_url: "{{ deploy_container.docker_registry_url }}"
+    username: "{{ _docker_registry_username }}"
+    password: "{{ _docker_registry_password }}"
+    reauthorize: true
+  delegate_to: localhost
+
+- name: Build and push container image.
+  community.docker.docker_image:
+    build:
+      path: "{{ deploy_container.docker_build_dir }}"
+    name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}"
+    tag: "{{ deploy_container.container_tag | default('latest') }}"
+    push: true
+    source: build
+  delegate_to: localhost
diff --git a/roles/deploy_container/templates/example.j2 b/roles/deploy_container/templates/example.j2
new file mode 100644
index 00000000..5a155197
--- /dev/null
+++ b/roles/deploy_container/templates/example.j2
@@ -0,0 +1,7 @@
+# Basic Dockerfile example
+FROM debian:bullseye-slim
+MAINTAINER sysadm@codeenigma.com 
+
+RUN apt-get update 
+RUN apt-get install -y nginx 
+CMD ["echo","Image created"]
\ No newline at end of file

From 1fa7eb47e5aa3c1662376388066d1af010d1403a Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 27 Jun 2023 18:19:34 +0200
Subject: [PATCH 069/139] Ecs deployments pr 1.x (#314)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.
---
 docs/roles/deploy_container.md           |  64 ++++-
 roles/deploy_container/README.md         |  64 ++++-
 roles/deploy_container/defaults/main.yml |  66 ++++-
 roles/deploy_container/tasks/main.yml    | 310 ++++++++++++++++++++++-
 roles/deploy_container/tasks/subnet.yml  |  13 +
 5 files changed, 507 insertions(+), 10 deletions(-)
 create mode 100644 roles/deploy_container/tasks/subnet.yml

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 9730b01e..d1befb03 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -31,7 +31,69 @@ deploy_container:
   aws_ecr:
     enabled: false # set to true if using AWS ECR
     region: eu-west-1
-    profile: example
+    aws_profile: example
+  # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached
+  # Note, you can if you wish make more restrictive roles and policies
+  aws_ecs:
+    enabled: false
+    region: eu-west-1
+    aws_profile: example
+    tags: {}
+    domain_name: www.example.com
+    route_53:
+      zone: example.com
+      aws_profile: example2 # might not be the same account
+    vpc_name: example
+    #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
+    subnets: # list of public subnet names
+      - example-dev-a
+      - example-dev-b
+    security_groups: [] # list of security groups, accepts names or IDs
+    cluster_name: example
+    family_name: example
+    task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_count: 1
+    task_minimum_count: 1
+    task_maximum_count: 4
+    # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
+    service_autoscale_metric_type: ECSServiceAverageCPUUtilization
+    service_autoscale_up_cooldown: 120
+    service_autoscale_down_cooldown: 120
+    service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    cpu: 512 # these values can be set globally or per container
+    memory: 1024
+    launch_type: FARGATE
+    network_mode: awsvpc
+    #volumes: [] # list of additional volumes to attach
+    target_group_name: example # 32 character limit
+    target_group_protocol: http
+    target_group_port: 80
+    target_group_wait_timeout: 200 # how long to wait for target group events to complete
+    targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
+      #- Id: 10.0.0.2
+      #  Port: 80
+      #  AvailabilityZone: all
+    health_check:
+      protocol: http
+      path: /
+      response_codes: "200"
+    # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
+    acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
+      create_cert: false
+      extra_domains: [] # list of Subject Alternative Name domains and zones
+    ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
+    elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_http_port: 80
+    elb_https_port: 443
+    elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
+    elb_listener_http_rules: []
+    elb_listener_https_rules: []
+    # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html
+    elb_listeners: []
+    elb_idle_timeout: 60
+    elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses).
 
 ```
 
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 9730b01e..d1befb03 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -31,7 +31,69 @@ deploy_container:
   aws_ecr:
     enabled: false # set to true if using AWS ECR
     region: eu-west-1
-    profile: example
+    aws_profile: example
+  # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached
+  # Note, you can if you wish make more restrictive roles and policies
+  aws_ecs:
+    enabled: false
+    region: eu-west-1
+    aws_profile: example
+    tags: {}
+    domain_name: www.example.com
+    route_53:
+      zone: example.com
+      aws_profile: example2 # might not be the same account
+    vpc_name: example
+    #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
+    subnets: # list of public subnet names
+      - example-dev-a
+      - example-dev-b
+    security_groups: [] # list of security groups, accepts names or IDs
+    cluster_name: example
+    family_name: example
+    task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_count: 1
+    task_minimum_count: 1
+    task_maximum_count: 4
+    # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
+    service_autoscale_metric_type: ECSServiceAverageCPUUtilization
+    service_autoscale_up_cooldown: 120
+    service_autoscale_down_cooldown: 120
+    service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    cpu: 512 # these values can be set globally or per container
+    memory: 1024
+    launch_type: FARGATE
+    network_mode: awsvpc
+    #volumes: [] # list of additional volumes to attach
+    target_group_name: example # 32 character limit
+    target_group_protocol: http
+    target_group_port: 80
+    target_group_wait_timeout: 200 # how long to wait for target group events to complete
+    targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
+      #- Id: 10.0.0.2
+      #  Port: 80
+      #  AvailabilityZone: all
+    health_check:
+      protocol: http
+      path: /
+      response_codes: "200"
+    # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
+    acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
+      create_cert: false
+      extra_domains: [] # list of Subject Alternative Name domains and zones
+    ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
+    elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_http_port: 80
+    elb_https_port: 443
+    elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
+    elb_listener_http_rules: []
+    elb_listener_https_rules: []
+    # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html
+    elb_listeners: []
+    elb_idle_timeout: 60
+    elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses).
 
 ```
 
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index 3578e12a..77be7a6a 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -8,9 +8,71 @@ deploy_container:
   docker_base_command: "docker image build"
   docker_build_dir: "{{ _ce_deploy_build_dir }}"
   dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
-  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template
   # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
   aws_ecr:
     enabled: false # set to true if using AWS ECR
     region: eu-west-1
-    profile: example
+    aws_profile: example
+  # Requires the deploy IAM user to have the managed AmazonECS_FullAccess and ElasticLoadBalancingFullAccess policies attached
+  # Note, you can if you wish make more restrictive roles and policies
+  aws_ecs:
+    enabled: false
+    region: eu-west-1
+    aws_profile: example
+    tags: {}
+    domain_name: www.example.com
+    route_53:
+      zone: example.com
+      aws_profile: example2 # might not be the same account
+    vpc_name: example
+    #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
+    subnets: # list of public subnet names
+      - example-dev-a
+      - example-dev-b
+    security_groups: [] # list of security groups, accepts names or IDs
+    cluster_name: example
+    family_name: example
+    task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_count: 1
+    task_minimum_count: 1
+    task_maximum_count: 4
+    # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
+    service_autoscale_metric_type: ECSServiceAverageCPUUtilization
+    service_autoscale_up_cooldown: 120
+    service_autoscale_down_cooldown: 120
+    service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    cpu: 512 # these values can be set globally or per container
+    memory: 1024
+    launch_type: FARGATE
+    network_mode: awsvpc
+    #volumes: [] # list of additional volumes to attach
+    target_group_name: example # 32 character limit
+    target_group_protocol: http
+    target_group_port: 80
+    target_group_wait_timeout: 200 # how long to wait for target group events to complete
+    targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
+      #- Id: 10.0.0.2
+      #  Port: 80
+      #  AvailabilityZone: all
+    health_check:
+      protocol: http
+      path: /
+      response_codes: "200"
+    # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
+    acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
+      create_cert: false
+      extra_domains: [] # list of Subject Alternative Name domains and zones
+    ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
+    elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_http_port: 80
+    elb_https_port: 443
+    elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
+    elb_listener_http_rules: []
+    elb_listener_https_rules: []
+    # Add custom listeners. See https://docs.ansible.com/ansible/latest/collections/amazon/aws/elb_application_lb_module.html
+    elb_listeners: []
+    elb_idle_timeout: 60
+    elb_ip_address_type: "ipv4" # Can be 'ipv4' or 'dualstack' (the latter includes IPv4 and IPv6 addresses).
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index c58bb8b1..23f4d996 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -1,10 +1,10 @@
 ---
-# @TODO - for AWS ECR we'll need certain policies attaching to the deploy IAM user
+# Build and ship a container image
 - name: Create Dockerfile from template.
-  local_action:
-    module: ansible.builtin.template
+  ansible.builtin.template:
     src: "{{ deploy_container.dockerfile_template }}"
     dest: "{{ deploy_container.docker_build_dir }}/Dockerfile"
+  delegate_to: localhost
 
 - name: Set Docker registry username and password.
   ansible.builtin.set_fact:
@@ -12,10 +12,9 @@
     _docker_registry_password: "{{ deploy_container.docker_registry_pass }}"
   delegate_to: localhost
 
-# Token valid for 12 hours
-- name: Fetch AWS ECR registry login token.
+- name: Fetch AWS ECR registry login token. # token valid for 12 hours
   ansible.builtin.command:
-    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.profile }}"
+    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}"
   when: deploy_container.aws_ecr.enabled
   delegate_to: localhost
   register: _docker_registry_ecr_token
@@ -32,6 +31,12 @@
   when: deploy_container.aws_ecr.enabled
   delegate_to: localhost
 
+- name: Remove Docker credentials file.
+  ansible.builtin.file:
+    state: absent
+    path: "/home/{{ deploy_user }}/.docker/config.json"
+  delegate_to: localhost
+
 - name: Log into Docker registry.
   community.docker.docker_login:
     registry_url: "{{ deploy_container.docker_registry_url }}"
@@ -49,3 +54,296 @@
     push: true
     source: build
   delegate_to: localhost
+
+# Fetch the ACM role from ce-provision
+- name: Ensure the aws_acm directory exists.
+  ansible.builtin.file:
+    path: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}"
+    state: directory
+    mode: '0755'
+  delegate_to: localhost
+  with_items:
+    - tasks
+    - defaults
+
+- name: Fetch the aws_acm files.
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml"
+    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml"
+  delegate_to: localhost
+  with_items:
+    - tasks
+    - defaults
+
+- name: Fetch the aws_acm tasks.
+  ansible.builtin.get_url:
+    url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml
+    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml"
+  delegate_to: localhost
+
+# Gather all network information
+- name: Gather VPC information.
+  amazon.aws.ec2_vpc_net_info:
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    region: "{{ deploy_container.aws_ecs.region }}"
+    filters:
+      "tag:Name": "{{ deploy_container.aws_ecs.vpc_name }}"
+  register: _aws_ecs_cluster_vpc
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.vpc_name is defined
+    - deploy_container.aws_ecs.vpc_name | length > 0
+
+- name: Set the VPC id from name.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_vpc_id: "{{ _aws_ecs_cluster_vpc.vpcs[0].vpc_id }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.vpc_name is defined
+    - deploy_container.aws_ecs.vpc_name | length > 0
+
+- name: Use provided VPC id.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_vpc_id: "{{ deploy_container.aws_ecs.vpc_id }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - (deploy_container.aws_ecs.vpc_name is not defined or deploy_container.aws_ecs.vpc_name | length < 0)
+
+- name: Reset subnets lists.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_public_subnets_ids: []
+  when: deploy_container.aws_ecs.enabled
+
+- name: Construct list of public subnet IDs.
+  ansible.builtin.include_tasks: subnet.yml
+  with_items: "{{ deploy_container.aws_ecs.subnets }}"
+  loop_control:
+    loop_var: subnet
+  when: deploy_container.aws_ecs.enabled
+
+# Construct AWS supporting assets
+- name: Create task definition.
+  community.aws.ecs_taskdefinition:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    family: "{{ deploy_container.aws_ecs.family_name }}"
+    execution_role_arn: "{{ deploy_container.aws_ecs.execution_role_arn }}"
+    containers: "{{ deploy_container.aws_ecs.containers }}"
+    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
+    cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}"
+    memory: "{{ deploy_container.aws_ecs.memory | default(omit) }}"
+    state: present
+    network_mode: "{{ deploy_container.aws_ecs.network_mode }}"
+    volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}"
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create a target group with IP address targets.
+  community.aws.elb_target_group:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    protocol: "{{ deploy_container.aws_ecs.target_group_protocol }}"
+    port: "{{ deploy_container.aws_ecs.target_group_port }}"
+    vpc_id: "{{ _aws_ecs_cluster_vpc_id }}"
+    health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}"
+    health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}"
+    successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}"
+    target_type: ip
+    targets: "{{ deploy_container.aws_ecs.targets }}"
+    state: present
+    wait_timeout: "{{ deploy_container.aws_ecs.target_group_wait_timeout }}"
+    wait: true
+  register: _aws_ecs_target_group
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create SSL certificate for load balancer.
+  ansible.builtin.include_role:
+    name: aws_acm
+  vars:
+    aws_acm:
+      region: "{{ deploy_container.aws_ecs.region }}"
+      aws_profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+      tags: "{{ deploy_container.aws_ecs.tags }}"
+      export: false
+      domain_name: "{{ deploy_container.aws_ecs.domain_name }}"
+      extra_domains: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+      route_53:
+        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
+        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
+  when:
+    - deploy_container.aws_ecs.acm.create_cert
+    - deploy_container.aws_ecs.enabled
+
+- name: Default to provided SSL certificate ARN.
+  ansible.builtin.set_fact:
+    _ssl_certificate_ARN: "{{ deploy_container.aws_ecs.ssl_certificate_ARN }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: If provided, override SSL certificate ARN with the one received from ACM.
+  ansible.builtin.set_fact:
+    _ssl_certificate_ARN: "{{ aws_acm_certificate_arn }}"
+  when:
+    - deploy_container.aws_ecs.acm.create_cert
+    - deploy_container.aws_ecs.enabled
+
+- name: Define default ALB listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners_http:
+      Protocol: HTTP
+      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
+      DefaultActions:
+        - Type: forward
+          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
+      Rules: "{{ deploy_container.aws_ecs.elb_listener_http_rules }}"
+    _aws_ecs_cluster_listeners_redirect:
+      Protocol: HTTP
+      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
+      DefaultActions:
+        - Type: redirect
+          RedirectConfig:
+            Protocol: HTTPS
+            Host: "#{host}"
+            Query: "#{query}"
+            Path: "/#{path}"
+            Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
+            StatusCode: HTTP_301
+    _aws_ecs_cluster_listeners_https:
+      Protocol: HTTPS
+      Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
+      SslPolicy: "{{ deploy_container.aws_ecs.elb_ssl_policy }}"
+      Certificates:
+        - CertificateArn: "{{ _ssl_certificate_ARN }}"
+      DefaultActions:
+        - Type: forward
+          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
+      Rules: "{{ deploy_container.aws_ecs.elb_listener_https_rules }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Add HTTP listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_http ] }}"
+  when:
+    - _ssl_certificate_ARN | length < 1
+    - deploy_container.aws_ecs.enabled
+
+- name: Add HTTPS Listener.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_redirect, _aws_ecs_cluster_listeners_https ] }}"
+  when:
+    - _ssl_certificate_ARN | length > 1
+    - deploy_container.aws_ecs.enabled
+
+- name: Add custom Listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ _aws_ecs_cluster_listeners + deploy_container.aws_ecs.elb_listeners }}"
+  when:
+    - deploy_container.aws_ecs.elb_listeners is defined
+    - deploy_container.aws_ecs.elb_listeners | length
+    - deploy_container.aws_ecs.enabled
+
+- name: Create an ALB.
+  amazon.aws.elb_application_lb:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    state: present
+    tags: "{{ deploy_container.aws_ecs.tags }}"
+    subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
+    security_groups: "{{ deploy_container.aws_ecs.elb_security_groups }}"
+    listeners: "{{ _aws_ecs_cluster_listeners }}"
+    idle_timeout: "{{ deploy_container.aws_ecs.elb_idle_timeout }}"
+    ip_address_type: "{{ deploy_container.aws_ecs.elb_ip_address_type }}"
+  register: _aws_ecs_cluster_alb
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Set task definition name.
+  ansible.builtin.set_fact:
+    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Set task definition revision if applicable.
+  ansible.builtin.set_fact:
+    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}:{{ deploy_container.aws_ecs.task_definition_revision }}"
+  when:
+    - deploy_container.aws_ecs.task_definition_revision | length > 0
+    - deploy_container.aws_ecs.enabled
+
+- name: Create ECS service.
+  community.aws.ecs_service:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: present
+    name: "{{ deploy_container.aws_ecs.family_name }}"
+    cluster: "{{ deploy_container.aws_ecs.cluster_name }}"
+    task_definition: "{{ _aws_ecs_service_task_definition }}"
+    desired_count: "{{ deploy_container.aws_ecs.task_count }}"
+    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
+    platform_version: LATEST
+    load_balancers: # see https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_LoadBalancer.html
+      - containerName: "{{ deploy_container.container_name }}"
+        containerPort: "{{ deploy_container.aws_ecs.target_group_port }}"
+        targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}"
+    network_configuration:
+      subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
+      security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
+      assign_public_ip: true # must be true for now - details: https://stackoverflow.com/a/66802973
+    tags: "{{ deploy_container.aws_ecs.tags }}"
+    wait: true
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create target tracking scaling policy for ECS service.
+  community.aws.application_autoscaling_policy:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: present
+    policy_name: "{{ deploy_container.aws_ecs.family_name }}"
+    service_namespace: ecs
+    resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}"
+    scalable_dimension: ecs:service:DesiredCount
+    minimum_tasks: "{{ deploy_container.aws_ecs.task_minimum_count }}"
+    maximum_tasks: "{{ deploy_container.aws_ecs.task_maximum_count }}"
+    policy_type: TargetTrackingScaling
+    target_tracking_scaling_policy_configuration:
+      PredefinedMetricSpecification:
+        PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}"
+      ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}"
+      ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}"
+      DisableScaleIn: false
+      TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}"
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Initialise the domains loop var with main domain entry DNS settings.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains:
+      - domain: "{{ deploy_container.aws_ecs.domain_name }}"
+        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
+        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Add extra_domains so we can loop through DNS records.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}"
+  loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+  when:
+    - deploy_container.aws_ecs.acm.extra_domains | length > 0
+    - deploy_container.aws_ecs.enabled
+
+- name: Add DNS records in Route 53.
+  amazon.aws.route53:
+    state: present
+    profile: "{{ item.aws_profile }}"
+    zone: "{{ item.zone }}"
+    record: "{{ item.domain }}"
+    type: CNAME
+    value: "{{ _aws_ecs_cluster_alb.dns_name }}"
+    overwrite: true
+  loop: "{{ _aws_ecs_cluster_dns_all_domains }}"
+  when:
+    - deploy_container.aws_ecs.route_53.zone | length > 0
+    - deploy_container.aws_ecs.enabled
\ No newline at end of file
diff --git a/roles/deploy_container/tasks/subnet.yml b/roles/deploy_container/tasks/subnet.yml
new file mode 100644
index 00000000..a0701a36
--- /dev/null
+++ b/roles/deploy_container/tasks/subnet.yml
@@ -0,0 +1,13 @@
+- name: Gather public subnet information.
+  amazon.aws.ec2_vpc_subnet_info:
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    region: "{{ deploy_container.aws_ecs.region }}"
+    filters:
+      vpc-id: "{{ _aws_ecs_cluster_vpc_id }}"
+      tag:Name: "{{ subnet }}"
+  register: _aws_ecs_cluster_public_subnet
+  delegate_to: localhost
+
+- name: Add public subnet to the list.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_public_subnets_ids: "{{ _aws_ecs_cluster_public_subnets_ids + [ _aws_ecs_cluster_public_subnet.subnets[0].subnet_id ] }}"

From 49a308711da7036e768c9a77e1b17dd5c076e2e3 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 27 Jun 2023 19:32:35 +0300
Subject: [PATCH 070/139] db_import_speed_up (#291)

* db_import_speed_up

* db_import_speed_up_fixed

* db_import_speed_up_fix2

* db_import_speed_up_fix3

* db_import_speed_up_fix4

* db_import_speed_up_fix5_and_gzip

* db_import_speed_up_fix6

* db_import_speed_up_fix7

* db_import_speed_up_fix_typo

* db_import_speed_up_fix8

* db_import_speed_up_fix9

* db_import_speed_up_refactoring
---
 .../tasks/revert-dump.yml                     | 17 ++++++-
 .../database_sync-mysql/defaults/main.yml     |  5 +-
 .../database_sync-mysql/tasks/sync.yml        | 50 +++++++++++++++----
 3 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 80898e7d..38522af3 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -1,7 +1,22 @@
 ---
+- name: Unpack dump file.
+  ansible.builtin.unarchive:
+    src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+    dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql"
+    remote_src: true
+  run_once: true
+  when: previous_build_number > 0
+
 - name: Revert database from dump.
-  ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}"
+  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql"
   args:
     executable: /bin/bash
+  run_once: true
   when: previous_build_number > 0
+
+- name: Delete unpacked dump file.
+  ansible.builtin.file:
+    path: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql"
+    state: absent
   run_once: true
+  when: previous_build_number > 0
diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 48c02b8b..88143a7b 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -1,7 +1,8 @@
 ---
 mysql_sync:
-  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
-  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
+  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 7815868f..ac346444 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -35,9 +35,24 @@
     - database.target.asg is defined
     - database.target.asg | length > 0
 
+
+- name: Register bzip2 archive type vars.
+  ansible.builtin.set_fact:
+    archive_file_type: "bz2"
+    archival_command: "bzip2"
+  when: 
+    - mysql_sync.archival_method == 'bzip2'
+
+- name: Register gunzip archive type vars.
+  ansible.builtin.set_fact:
+    archive_file_type: "gz"
+    archival_command: "gzip"
+  when: 
+    - mysql_sync.archival_method == 'gzip'
+
 - name: Register remote dump name (from database).
   ansible.builtin.set_fact:
-    mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.bz2"
+    mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.{{ archive_file_type }}"
 
 - name: Get source last known good build number.
   ansible.builtin.command:
@@ -61,7 +76,7 @@
   when: not database.source.type == 'rolling'
 
 - name: Take a dump from source database.
-  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}"
+  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}"
   args:
     executable: /bin/bash
   delegate_to: "{{ database.source.host }}"
@@ -91,7 +106,11 @@
 
 - name: Register tmp target dump name.
   ansible.builtin.set_fact:
-    mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2"
+    mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.{{ archive_file_type }}"
+
+- name: Register tmp unpacked target dump name.
+  ansible.builtin.set_fact:
+    mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql"
 
 - name: Get target last known good build number.
   ansible.builtin.command:
@@ -117,15 +136,25 @@
 - name: Fetch dump file.
   ansible.builtin.fetch:
     src: "{{ mysql_sync_source_dump_path }}"
-    dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
+    dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}"
     flat: true
   delegate_to: "{{ database.source.host }}"
 
 - name: Copy dump file to destination.
   ansible.builtin.copy:
-    src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
+    src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}"
     dest: "{{ mysql_sync_target_dump_path }}"
 
+- name: Unpack dump file.
+  ansible.builtin.shell: "{{ archival_command }} -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}"
+  args:
+    executable: /bin/bash
+
+- name: Delete temporary dump file on target.
+  ansible.builtin.file:
+    path: "{{ mysql_sync_target_dump_path }}"
+    state: absent
+
 - name: Drop target database.
   ansible.builtin.command:
     cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'"
@@ -135,13 +164,13 @@
     cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
 
 - name: Repopulate database from dump.
-  ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}"
+  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}"
   args:
     executable: /bin/bash
 
-- name: Delete temporary dump file on target.
+- name: Delete temporary unpacked dump file on target.
   ansible.builtin.file:
-    path: "{{ mysql_sync_target_dump_path }}"
+    path: "{{ mysql_sync_target_dump_unpacked_path }}"
     state: absent
 
 - name: Delete temporary dump file on source.
@@ -152,11 +181,14 @@
 
 - name: Delete temporary dump file on deploy server.
   ansible.builtin.file:
-    path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2"
+    path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}"
     state: absent
   delegate_to: localhost
   when:
     - mysql_sync.cleanup
+  with_items:
+    - ".bz2"
+    - ".gz"
 
 - name: Enable all autoscale processes on source ASG.
   ansible.builtin.command: >

From fc3b544aa3cee288bfcec7efc5e31ef35427c5f8 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 27 Jun 2023 18:42:01 +0200
Subject: [PATCH 071/139] Ecs deployments pr 1.x (#317)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.
---
 docs/roles/deploy_container.md           | 3 ++-
 roles/deploy_container/README.md         | 3 ++-
 roles/deploy_container/defaults/main.yml | 1 +
 roles/deploy_container/tasks/main.yml    | 2 ++
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index d1befb03..8f7dca28 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -20,13 +20,14 @@ AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the
 deploy_container:
   container_name: example
   container_tag: latest # tag will take format container_name:container_tag
+  container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
   docker_registry_user: example
   docker_registry_pass: asdf1234
   docker_base_command: "docker image build"
   docker_build_dir: "{{ _ce_deploy_build_dir }}"
   dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
-  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template
   # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
   aws_ecr:
     enabled: false # set to true if using AWS ECR
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index d1befb03..8f7dca28 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -20,13 +20,14 @@ AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the
 deploy_container:
   container_name: example
   container_tag: latest # tag will take format container_name:container_tag
+  container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
   docker_registry_user: example
   docker_registry_pass: asdf1234
   docker_base_command: "docker image build"
   docker_build_dir: "{{ _ce_deploy_build_dir }}"
   dockerfile_template: example.j2 # provide a templates directory next to your playbook and change this to match your Dockerfile template name
-  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template   
+  environment_vars: {} # dictionary you can populate for use in a custom Dockerfile template
   # Requires the deploy IAM user to have the managed EC2InstanceProfileForImageBuilderECRContainerBuilds policy attached
   aws_ecr:
     enabled: false # set to true if using AWS ECR
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index 77be7a6a..7605f796 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -2,6 +2,7 @@
 deploy_container:
   container_name: example
   container_tag: latest # tag will take format container_name:container_tag
+  container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
   docker_registry_user: example
   docker_registry_pass: asdf1234
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index 23f4d996..34866aa1 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -53,6 +53,8 @@
     tag: "{{ deploy_container.container_tag | default('latest') }}"
     push: true
     source: build
+    force_source: "{{ deploy_container.container_force_build }}"
+    force_tag: "{{ deploy_container.container_force_build }}"
   delegate_to: localhost
 
 # Fetch the ACM role from ce-provision

From 014874bf86ce58e60817e70949a6c73a10d2ee84 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 27 Jun 2023 20:07:29 +0300
Subject: [PATCH 072/139] fix_typo (#320)

* fix_typo

* fix_typo
---
 roles/cache_clear/cache_clear-drupal8/tasks/main.yml        | 1 -
 roles/database_apply/database_apply-drupal8/tasks/main.yml  | 1 -
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 4 ++--
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index e9524b81..5289f254 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -15,4 +15,3 @@
   ansible.builtin.debug:
     msg: "{{ _drush_output }}"
   when: drupal.drush_verbose_output
-
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index e2fecd17..c423fad9 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -85,4 +85,3 @@
   ansible.builtin.debug:
     msg: "{{ _drush_output }}"
   when: drupal.drush_verbose_output
-
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index ac346444..7e52bd07 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -40,14 +40,14 @@
   ansible.builtin.set_fact:
     archive_file_type: "bz2"
     archival_command: "bzip2"
-  when: 
+  when:
     - mysql_sync.archival_method == 'bzip2'
 
 - name: Register gunzip archive type vars.
   ansible.builtin.set_fact:
     archive_file_type: "gz"
     archival_command: "gzip"
-  when: 
+  when:
     - mysql_sync.archival_method == 'gzip'
 
 - name: Register remote dump name (from database).

From 9b5d56bd64158058e74cdbe29db90f76843cd075 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 27 Jun 2023 19:12:25 +0200
Subject: [PATCH 073/139] Ecs deployments pr 1.x (#322)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.

* Improving docs for deploy_container role.
---
 docs/roles/deploy_container.md                | 22 ++++++++++++++++++-
 .../sync/database_sync/database_sync-mysql.md |  5 +++--
 roles/deploy_container/README.md              | 22 ++++++++++++++++++-
 .../database_sync-mysql/README.md             |  5 +++--
 4 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 8f7dca28..3e5a9c30 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -8,7 +8,27 @@ sudo usermod -aG docker deploy
 
 This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
 
-AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+## AWS IAM requirements
+AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached.
+
+If you enable AWS ECR registry integration by setting `deploy_container.aws_ecr.enabled` to `true` then you will need the `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+
+Similarly, if you set `deploy_container.aws_ecs.acm.create_cert` to `true` then you will need the `AWSCertificateManagerFullAccess` policy attaching to create SSL certificates.
+
+If you enable full AWS ECS integration by setting `deploy_container.aws_ecs.enabled` to `true` then this requires the following policies to be attached to the AWS CLI user:
+* `AmazonECS_FullAccess` - to create task definitions and services
+* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups
+
+Finally, if you set `deploy_container.aws_ecs.route_53.zone` to another other than an empty string then you will also need `AmazonRoute53FullAccess` attaching to manipulate DNS entries in Route 53.
+
+The full list is:
+* `EC2InstanceProfileForImageBuilderECRContainerBuilds` - to manipulate images in AWS ECR
+* `AWSCertificateManagerFullAccess` - to manage SSL certificates
+* `AmazonECS_FullAccess` - to create task definitions and services
+* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups
+* `AmazonRoute53FullAccess` - to manage DNS entries
+
+Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role.
 
 <!--TOC-->
 <!--ENDTOC-->
diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index cbe1a353..d70d770e 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -5,8 +5,9 @@ Sync MySQL databases between environments.
 ```yaml
 ---
 mysql_sync:
-  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
-  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
+  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 8f7dca28..3e5a9c30 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -8,7 +8,27 @@ sudo usermod -aG docker deploy
 
 This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
 
-AWS ECR registries require the AWS CLI user provided for `ce-deploy` to have the managed AWS `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+## AWS IAM requirements
+AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached.
+
+If you enable AWS ECR registry integration by setting `deploy_container.aws_ecr.enabled` to `true` then you will need the `EC2InstanceProfileForImageBuilderECRContainerBuilds` policy attached via IAM to allow access to fetch credentials and push containers.
+
+Similarly, if you set `deploy_container.aws_ecs.acm.create_cert` to `true` then you will need the `AWSCertificateManagerFullAccess` policy attaching to create SSL certificates.
+
+If you enable full AWS ECS integration by setting `deploy_container.aws_ecs.enabled` to `true` then this requires the following policies to be attached to the AWS CLI user:
+* `AmazonECS_FullAccess` - to create task definitions and services
+* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups
+
+Finally, if you set `deploy_container.aws_ecs.route_53.zone` to another other than an empty string then you will also need `AmazonRoute53FullAccess` attaching to manipulate DNS entries in Route 53.
+
+The full list is:
+* `EC2InstanceProfileForImageBuilderECRContainerBuilds` - to manipulate images in AWS ECR
+* `AWSCertificateManagerFullAccess` - to manage SSL certificates
+* `AmazonECS_FullAccess` - to create task definitions and services
+* `ElasticLoadBalancingFullAccess` - to create load balancers and target groups
+* `AmazonRoute53FullAccess` - to manage DNS entries
+
+Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role.
 
 <!--TOC-->
 <!--ENDTOC-->
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index cbe1a353..d70d770e 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -5,8 +5,9 @@ Sync MySQL databases between environments.
 ```yaml
 ---
 mysql_sync:
-  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
-  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes
+  mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
+  cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
+  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.

From 77087edd6ebbfdb5f17cece0bddcd5a50994f0a7 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 29 Jun 2023 14:54:08 +0200
Subject: [PATCH 074/139] Ecs deployments pr 1.x (#325)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.

* Improving docs for deploy_container role.

* Providing container description example for ECS.

* Adding 'force' option to docker builds.

* Revert "Adding 'force' option to docker builds."

This reverts commit 5574faa182f023c20ec34ae6dc7125c92b4c7dcb.

* Adding options to force ECS component refreshes.
---
 docs/roles/deploy_container.md           | 24 +++++++++++++++++++-----
 roles/deploy_container/README.md         | 24 +++++++++++++++++++-----
 roles/deploy_container/defaults/main.yml | 24 +++++++++++++++++++-----
 roles/deploy_container/tasks/main.yml    |  2 ++
 4 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 3e5a9c30..8e5872cb 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -38,7 +38,7 @@ Naturally you can always create custom policies and roles to have tighter access
 ```yaml
 ---
 deploy_container:
-  container_name: example
+  container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
@@ -70,9 +70,10 @@ deploy_container:
       - example-dev-a
       - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
-    cluster_name: example
-    family_name: example
+    cluster_name: example-cluster
+    family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_definition_force_create: false # creates a task definition revision every time if set to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
@@ -81,8 +82,21 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
-    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+      - name: example-container
+        essential: true
+        image: index.docker.io/example:latest
+        portMappings:
+          - containerPort: 8080 # should match target_group_port
+            hostPort: 8080
+        logConfiguration:
+          logDriver: awslogs
+          options:
+            awslogs-group: /ecs/example-cluster
+            awslogs-region: eu-west-1
+            awslogs-stream-prefix: "ecs-example-task"
     cpu: 512 # these values can be set globally or per container
     memory: 1024
     launch_type: FARGATE
@@ -90,7 +104,7 @@ deploy_container:
     #volumes: [] # list of additional volumes to attach
     target_group_name: example # 32 character limit
     target_group_protocol: http
-    target_group_port: 80
+    target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
     targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
       #- Id: 10.0.0.2
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 3e5a9c30..8e5872cb 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -38,7 +38,7 @@ Naturally you can always create custom policies and roles to have tighter access
 ```yaml
 ---
 deploy_container:
-  container_name: example
+  container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
@@ -70,9 +70,10 @@ deploy_container:
       - example-dev-a
       - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
-    cluster_name: example
-    family_name: example
+    cluster_name: example-cluster
+    family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_definition_force_create: false # creates a task definition revision every time if set to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
@@ -81,8 +82,21 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
-    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+      - name: example-container
+        essential: true
+        image: index.docker.io/example:latest
+        portMappings:
+          - containerPort: 8080 # should match target_group_port
+            hostPort: 8080
+        logConfiguration:
+          logDriver: awslogs
+          options:
+            awslogs-group: /ecs/example-cluster
+            awslogs-region: eu-west-1
+            awslogs-stream-prefix: "ecs-example-task"
     cpu: 512 # these values can be set globally or per container
     memory: 1024
     launch_type: FARGATE
@@ -90,7 +104,7 @@ deploy_container:
     #volumes: [] # list of additional volumes to attach
     target_group_name: example # 32 character limit
     target_group_protocol: http
-    target_group_port: 80
+    target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
     targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
       #- Id: 10.0.0.2
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index 7605f796..0ca73b42 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -1,6 +1,6 @@
 ---
 deploy_container:
-  container_name: example
+  container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
   docker_registry_name: index.docker.io/example # combines with container_name to make the full registry name, docker_registry_name/container_name
@@ -32,9 +32,10 @@ deploy_container:
       - example-dev-a
       - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
-    cluster_name: example
-    family_name: example
+    cluster_name: example-cluster
+    family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
+    task_definition_force_create: false # creates a task definition revision every time if set to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
@@ -43,8 +44,21 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
-    containers: [] # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+    containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
+      - name: example-container
+        essential: true
+        image: index.docker.io/example:latest
+        portMappings:
+          - containerPort: 8080 # should match target_group_port
+            hostPort: 8080
+        logConfiguration:
+          logDriver: awslogs
+          options:
+            awslogs-group: /ecs/example-cluster
+            awslogs-region: eu-west-1
+            awslogs-stream-prefix: "ecs-example-task"
     cpu: 512 # these values can be set globally or per container
     memory: 1024
     launch_type: FARGATE
@@ -52,7 +66,7 @@ deploy_container:
     #volumes: [] # list of additional volumes to attach
     target_group_name: example # 32 character limit
     target_group_protocol: http
-    target_group_port: 80
+    target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
     targets: [] # typically we do not specify targets at this point, this will be handled automatically by the ECS service
       #- Id: 10.0.0.2
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index 34866aa1..8238e080 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -138,6 +138,7 @@
     state: present
     network_mode: "{{ deploy_container.aws_ecs.network_mode }}"
     volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}"
+    force_create: "{{ deploy_container.aws_ecs.task_definition_force_create }}"
   delegate_to: localhost
   when: deploy_container.aws_ecs.enabled
 
@@ -294,6 +295,7 @@
       security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
       assign_public_ip: true # must be true for now - details: https://stackoverflow.com/a/66802973
     tags: "{{ deploy_container.aws_ecs.tags }}"
+    force_new_deployment: "{{ deploy_container.aws_ecs.service_force_refresh }}"
     wait: true
   delegate_to: localhost
   when: deploy_container.aws_ecs.enabled

From 443fdf51f0d2583c8a4a74fa6cb34764a3f99a13 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 29 Jun 2023 18:54:17 +0200
Subject: [PATCH 075/139] Ecs deployments pr 1.x (#329)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.

* Improving docs for deploy_container role.

* Providing container description example for ECS.

* Adding 'force' option to docker builds.

* Revert "Adding 'force' option to docker builds."

This reverts commit 5574faa182f023c20ec34ae6dc7125c92b4c7dcb.

* Adding options to force ECS component refreshes.

* Allowing users to toggle public IP and execute command mode.
---
 docs/roles/deploy_container.md           | 2 ++
 roles/deploy_container/README.md         | 2 ++
 roles/deploy_container/defaults/main.yml | 2 ++
 roles/deploy_container/tasks/main.yml    | 3 ++-
 4 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 8e5872cb..9970dc39 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -82,6 +82,8 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
+    service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 8e5872cb..9970dc39 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -82,6 +82,8 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
+    service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index 0ca73b42..a9b7e9d9 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -44,6 +44,8 @@ deploy_container:
     service_autoscale_up_cooldown: 120
     service_autoscale_down_cooldown: 120
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
+    service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
+    service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
     execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index 8238e080..de12e5bc 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -293,8 +293,9 @@
     network_configuration:
       subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
       security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
-      assign_public_ip: true # must be true for now - details: https://stackoverflow.com/a/66802973
+      assign_public_ip: "{{ deploy_container.aws_ecs.service_public_container_ip }}"
     tags: "{{ deploy_container.aws_ecs.tags }}"
+    enable_execute_command: "{{ deploy_container.aws_ecs.service_enable_ssm }}"
     force_new_deployment: "{{ deploy_container.aws_ecs.service_force_refresh }}"
     wait: true
   delegate_to: localhost

From 0859ebf7b1aa9995816bc549cbe770c9d5b98484 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 30 Jun 2023 11:17:27 +0200
Subject: [PATCH 076/139] Ecs deployments pr 1.x (#331)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.

* Improving docs for deploy_container role.

* Providing container description example for ECS.

* Adding 'force' option to docker builds.

* Revert "Adding 'force' option to docker builds."

This reverts commit 5574faa182f023c20ec34ae6dc7125c92b4c7dcb.

* Adding options to force ECS component refreshes.

* Allowing users to toggle public IP and execute command mode.

* The ALB needs to be on public subnets, not the same private ones as the cluster.

* Updating documentation.

* Updating ECS docs.
---
 docs/roles/deploy_container.md                  | 16 ++++++++++++----
 roles/deploy_container/README.md                | 16 ++++++++++++----
 roles/deploy_container/defaults/main.yml        | 13 +++++++++----
 roles/deploy_container/tasks/main.yml           | 17 +++++++++++++----
 roles/deploy_container/tasks/subnet-private.yml | 13 +++++++++++++
 .../tasks/{subnet.yml => subnet-public.yml}     |  0
 6 files changed, 59 insertions(+), 16 deletions(-)
 create mode 100644 roles/deploy_container/tasks/subnet-private.yml
 rename roles/deploy_container/tasks/{subnet.yml => subnet-public.yml} (100%)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 9970dc39..f127bb61 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -30,6 +30,9 @@ The full list is:
 
 Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role.
 
+# Peculiarities of AWS ECS
+It is worth noting that even if you put your containers on private subnets and configure your apps to use internal addressing, traffic will pass via the public interface. Therefore any safelisting of IP addresses needs to include the IP addresses of the NAT gateways of your private subnets. [More on how this works here.](https://docs.aws.amazon.com/AmazonECS/latest/bestpracticesguide/networking-connecting-vpc.html)
+
 <!--TOC-->
 <!--ENDTOC-->
 
@@ -66,17 +69,20 @@ deploy_container:
       aws_profile: example2 # might not be the same account
     vpc_name: example
     #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
-    subnets: # list of public subnet names
-      - example-dev-a
-      - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
     cluster_name: example-cluster
     family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
     task_definition_force_create: false # creates a task definition revision every time if set to true
+    task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    #task_role_arn: "" # required if you set service_enable_ssm to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
+    # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access.
+    service_subnets: # list of private subnet names
+      - example-cluster-dev-a
+      - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
     service_autoscale_up_cooldown: 120
@@ -85,7 +91,6 @@ deploy_container:
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
-    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
       - name: example-container
         essential: true
@@ -122,6 +127,9 @@ deploy_container:
       extra_domains: [] # list of Subject Alternative Name domains and zones
     ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
     elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_subnets: # must be public subnets for public facing applications
+      - example-dev-a
+      - example-dev-b
     elb_http_port: 80
     elb_https_port: 443
     elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 9970dc39..f127bb61 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -30,6 +30,9 @@ The full list is:
 
 Naturally you can always create custom policies and roles to have tighter access control. This document simply gives you the broad strokes AWS managed policies you can use in conjunction with this Ansible role.
 
+# Peculiarities of AWS ECS
+It is worth noting that even if you put your containers on private subnets and configure your apps to use internal addressing, traffic will pass via the public interface. Therefore any safelisting of IP addresses needs to include the IP addresses of the NAT gateways of your private subnets. [More on how this works here.](https://docs.aws.amazon.com/AmazonECS/latest/bestpracticesguide/networking-connecting-vpc.html)
+
 <!--TOC-->
 <!--ENDTOC-->
 
@@ -66,17 +69,20 @@ deploy_container:
       aws_profile: example2 # might not be the same account
     vpc_name: example
     #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
-    subnets: # list of public subnet names
-      - example-dev-a
-      - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
     cluster_name: example-cluster
     family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
     task_definition_force_create: false # creates a task definition revision every time if set to true
+    task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    #task_role_arn: "" # required if you set service_enable_ssm to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
+    # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access.
+    service_subnets: # list of private subnet names
+      - example-cluster-dev-a
+      - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
     service_autoscale_up_cooldown: 120
@@ -85,7 +91,6 @@ deploy_container:
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
-    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
       - name: example-container
         essential: true
@@ -122,6 +127,9 @@ deploy_container:
       extra_domains: [] # list of Subject Alternative Name domains and zones
     ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
     elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_subnets: # must be public subnets for public facing applications
+      - example-dev-a
+      - example-dev-b
     elb_http_port: 80
     elb_https_port: 443
     elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index a9b7e9d9..be178aca 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -28,17 +28,20 @@ deploy_container:
       aws_profile: example2 # might not be the same account
     vpc_name: example
     #vpc_id: vpc-XXXXXXX # optionally specify VPC ID to use
-    subnets: # list of public subnet names
-      - example-dev-a
-      - example-dev-b
     security_groups: [] # list of security groups, accepts names or IDs
     cluster_name: example-cluster
     family_name: example-task-definition
     task_definition_revision: "" # integer, but must be presented as a string for Jinja2
     task_definition_force_create: false # creates a task definition revision every time if set to true
+    task_execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
+    #task_role_arn: "" # required if you set service_enable_ssm to true
     task_count: 1
     task_minimum_count: 1
     task_maximum_count: 4
+    # These subnets are usually the subnets created by ce-provision when you made your ECS cluster and must have a NAT gateway for ECR access.
+    service_subnets: # list of private subnet names
+      - example-cluster-dev-a
+      - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
     service_autoscale_up_cooldown: 120
@@ -47,7 +50,6 @@ deploy_container:
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
     service_force_refresh: false # forces a refresh of all containers if set to true
-    execution_role_arn: "arn:aws:iam::000000000000:role/ecsTaskExecutionRole" # ARN of the IAM role to run the task as, must have access to the ECR repository if applicable
     containers: # list of container definitions, see docs: https://docs.ansible.com/ansible/latest/collections/community/aws/ecs_taskdefinition_module.html#parameter-containers
       - name: example-container
         essential: true
@@ -84,6 +86,9 @@ deploy_container:
       extra_domains: [] # list of Subject Alternative Name domains and zones
     ssl_certificate_ARN: "" # optional SSL cert ARN if you imported one into AWS Certificate Manager
     elb_security_groups: [] # default SG is used if none provided - module supports names or IDs
+    elb_subnets: # must be public subnets for public facing applications
+      - example-dev-a
+      - example-dev-b
     elb_http_port: 80
     elb_https_port: 443
     elb_ssl_policy: ELBSecurityPolicy-TLS13-1-2-2021-06 # see https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index de12e5bc..6c68bebd 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -115,11 +115,19 @@
 - name: Reset subnets lists.
   ansible.builtin.set_fact:
     _aws_ecs_cluster_public_subnets_ids: []
+    _aws_ecs_cluster_private_subnets_ids: []
   when: deploy_container.aws_ecs.enabled
 
 - name: Construct list of public subnet IDs.
-  ansible.builtin.include_tasks: subnet.yml
-  with_items: "{{ deploy_container.aws_ecs.subnets }}"
+  ansible.builtin.include_tasks: subnet-public.yml
+  with_items: "{{ deploy_container.aws_ecs.elb_subnets }}"
+  loop_control:
+    loop_var: subnet
+  when: deploy_container.aws_ecs.enabled
+
+- name: Construct list of private subnet IDs.
+  ansible.builtin.include_tasks: subnet-private.yml
+  with_items: "{{ deploy_container.aws_ecs.service_subnets }}"
   loop_control:
     loop_var: subnet
   when: deploy_container.aws_ecs.enabled
@@ -130,7 +138,8 @@
     region: "{{ deploy_container.aws_ecs.region }}"
     profile: "{{ deploy_container.aws_ecs.aws_profile }}"
     family: "{{ deploy_container.aws_ecs.family_name }}"
-    execution_role_arn: "{{ deploy_container.aws_ecs.execution_role_arn }}"
+    execution_role_arn: "{{ deploy_container.aws_ecs.task_execution_role_arn }}"
+    task_role_arn: "{{ deploy_container.aws_ecs.task_role_arn | default(omit) }}"
     containers: "{{ deploy_container.aws_ecs.containers }}"
     launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
     cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}"
@@ -291,7 +300,7 @@
         containerPort: "{{ deploy_container.aws_ecs.target_group_port }}"
         targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}"
     network_configuration:
-      subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
+      subnets: "{{ _aws_ecs_cluster_private_subnets_ids }}" # internal private subnet
       security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
       assign_public_ip: "{{ deploy_container.aws_ecs.service_public_container_ip }}"
     tags: "{{ deploy_container.aws_ecs.tags }}"
diff --git a/roles/deploy_container/tasks/subnet-private.yml b/roles/deploy_container/tasks/subnet-private.yml
new file mode 100644
index 00000000..2335cb89
--- /dev/null
+++ b/roles/deploy_container/tasks/subnet-private.yml
@@ -0,0 +1,13 @@
+- name: Gather private subnet information.
+  amazon.aws.ec2_vpc_subnet_info:
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    region: "{{ deploy_container.aws_ecs.region }}"
+    filters:
+      vpc-id: "{{ _aws_ecs_cluster_vpc_id }}"
+      tag:Name: "{{ subnet }}"
+  register: _aws_ecs_cluster_private_subnet
+  delegate_to: localhost
+
+- name: Add private subnet to the list.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_private_subnets_ids: "{{ _aws_ecs_cluster_private_subnets_ids + [ _aws_ecs_cluster_private_subnet.subnets[0].subnet_id ] }}"
diff --git a/roles/deploy_container/tasks/subnet.yml b/roles/deploy_container/tasks/subnet-public.yml
similarity index 100%
rename from roles/deploy_container/tasks/subnet.yml
rename to roles/deploy_container/tasks/subnet-public.yml

From ace4578cea247f3d7ecc5ab91c57059cab6eb76d Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 10 Jul 2023 11:47:18 +0200
Subject: [PATCH 077/139] Bug fixes pr 1.x (#334)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.
---
 .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 38522af3..4fee5ceb 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -2,7 +2,7 @@
 - name: Unpack dump file.
   ansible.builtin.unarchive:
     src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
-    dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql"
+    dest: "/tmp"
     remote_src: true
   run_once: true
   when: previous_build_number > 0

From 0df95e015c8950b4dcb29af679ada9cae7702f87 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 19 Jul 2023 16:07:55 +0200
Subject: [PATCH 078/139] Bug fixes pr 1.x (#336)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.
---
 .github/workflows/ce-deploy-lint.yml          | 28 ++++++++
 ...oy-test.yml => ce-deploy-publish-docs.yml} | 64 ++++---------------
 .github/workflows/ce-deploy-test-drupal8.yml  | 44 +++++++++++++
 .github/workflows/ce-deploy-test-drupal9.yml  | 44 +++++++++++++
 ce-dev/ansible/common.yml                     |  2 +-
 ce-dev/ansible/examples/drupal8/deploy.yml    | 33 ++++++++--
 ce-dev/ansible/examples/drupal9/deploy.yml    | 13 ++--
 ce-dev/ansible/examples/localgov/deploy.yml   | 49 ++++++++++----
 8 files changed, 201 insertions(+), 76 deletions(-)
 create mode 100644 .github/workflows/ce-deploy-lint.yml
 rename .github/workflows/{ce-deploy-test.yml => ce-deploy-publish-docs.yml} (61%)
 create mode 100644 .github/workflows/ce-deploy-test-drupal8.yml
 create mode 100644 .github/workflows/ce-deploy-test-drupal9.yml

diff --git a/.github/workflows/ce-deploy-lint.yml b/.github/workflows/ce-deploy-lint.yml
new file mode 100644
index 00000000..283cd3b9
--- /dev/null
+++ b/.github/workflows/ce-deploy-lint.yml
@@ -0,0 +1,28 @@
+name: Linting
+
+# Run this workflow every time a new commit pushed to your repository
+on: pull_request
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  run-linting:
+    # Name the Job
+    name: Linting
+    # Set the type of machine to run on
+    runs-on: ubuntu-20.04
+
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # Linter checks.
+      - name: Run linters.
+        if: ${{ always() }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ansible-lint yamllint shellcheck
+          find ./roles -name "*.yml"  | xargs ansible-lint
+          yamllint ./roles
+          cd scripts && shellcheck *.sh
diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-publish-docs.yml
similarity index 61%
rename from .github/workflows/ce-deploy-test.yml
rename to .github/workflows/ce-deploy-publish-docs.yml
index b93a6d3e..45148600 100644
--- a/.github/workflows/ce-deploy-test.yml
+++ b/.github/workflows/ce-deploy-publish-docs.yml
@@ -1,14 +1,18 @@
-name: Run tests and build docs
+name: Publish documentation
 
 # Run this workflow every time a new commit pushed to your repository
-on: pull_request
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - 1.x
 
 jobs:
   # Set the job key. The key is displayed as the job name
   # when a job name is not provided
-  run-tests:
+  publish-docs:
     # Name the Job
-    name: Run tests against Ansible code base
+    name: Publish ce-deploy documentation
     # Set the type of machine to run on
     runs-on: ubuntu-20.04
 
@@ -17,15 +21,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
 
-      # Linter checks.
-      - name: Run linters.
-        if: ${{ always() }}
+      # Configures global Git variables for committing
+      - name: Configure Git
         run: |
-          sudo apt-get update
-          sudo apt-get install -y ansible-lint yamllint shellcheck
-          find ./roles -name "*.yml"  | xargs ansible-lint
-          yamllint ./roles
-          cd scripts && shellcheck *.sh
+          git config --global user.email "sysadm@codeenigma.com"
+          git config --global user.name "Code Enigma CI"
 
       # Installs the ce-dev stack
       - name: Install ce-dev
@@ -41,46 +41,6 @@ jobs:
           rm -Rf mkcert
           curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux
 
-      # Configures global Git variables for committing
-      - name: Configure Git
-        run: |
-          git config --global user.email "sysadm@codeenigma.com"
-          git config --global user.name "Code Enigma CI"
-
-      # Uses the ce-dev stack to run a test Drupal deploy
-      - name: Run a test ce-dev deploy with Drupal 8
-        run: |
-          ce-dev create -p test -t drupal8 -d ~/test
-          cd ~/test
-          ce-dev init
-          ce-dev start
-          ce-dev provision
-          ce-dev deploy
-          curl https://www.test.local
-        shell: bash
-
-      - name: Run a test ce-dev deploy with Drupal 9
-        run: |
-          ce-dev create -p testnine -t drupal9 -d ~/testnine
-          cd ~/testnine
-          ce-dev init
-          ce-dev start
-          ce-dev provision
-          ce-dev deploy
-          curl https://www.testnine.local
-        shell: bash
-
-      - name: Run a test ce-dev deploy with Localgov
-        run: |
-          ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov
-          cd ~/testlocalgov
-          ce-dev init
-          ce-dev start
-          ce-dev provision
-          ce-dev deploy
-          curl https://www.testlocalgov.local
-        shell: bash
-
       # Builds the table of contents for the docs
       - name: Documentation (build table of contents)
         if: ${{ github.event.pull_request.base.ref == '1.x' }}
diff --git a/.github/workflows/ce-deploy-test-drupal8.yml b/.github/workflows/ce-deploy-test-drupal8.yml
new file mode 100644
index 00000000..ec9909a9
--- /dev/null
+++ b/.github/workflows/ce-deploy-test-drupal8.yml
@@ -0,0 +1,44 @@
+name: Run test for Drupal 8
+
+# Run this workflow every time a new commit pushed to your repository
+on: pull_request
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  run-tests:
+    # Name the Job
+    name: Run Drupal 8 test build
+    # Set the type of machine to run on
+    runs-on: ubuntu-20.04
+
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # Installs the ce-dev stack
+      - name: Install ce-dev
+        run: |
+          cd /tmp
+          wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz
+          sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz
+          export PATH=$PATH:/usr/local/go/bin
+          git clone https://github.com/FiloSottile/mkcert && cd mkcert
+          go build -ldflags "-X main.Version=$(git describe --tags)"
+          sudo mv ./mkcert /usr/local/bin && cd ../
+          sudo chmod +x /usr/local/bin/mkcert
+          rm -Rf mkcert
+          curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux
+
+      # Uses the ce-dev stack to run a test Drupal deploy
+      - name: Run a test ce-dev deploy with Drupal 8
+        run: |
+          ce-dev create -p test -t drupal8 -d ~/test
+          cd ~/test
+          ce-dev init
+          ce-dev start
+          ce-dev provision
+          ce-dev deploy
+          curl https://www.test.local
+        shell: bash
diff --git a/.github/workflows/ce-deploy-test-drupal9.yml b/.github/workflows/ce-deploy-test-drupal9.yml
new file mode 100644
index 00000000..afaaca7a
--- /dev/null
+++ b/.github/workflows/ce-deploy-test-drupal9.yml
@@ -0,0 +1,44 @@
+name: Run test for Drupal 9
+
+# Run this workflow every time a new commit pushed to your repository
+on: pull_request
+
+jobs:
+  # Set the job key. The key is displayed as the job name
+  # when a job name is not provided
+  run-tests:
+    # Name the Job
+    name: Run Drupal 9 test build
+    # Set the type of machine to run on
+    runs-on: ubuntu-20.04
+
+    steps:
+      # Checks out a copy of your repository on the ubuntu-latest machine
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      # Installs the ce-dev stack
+      - name: Install ce-dev
+        run: |
+          cd /tmp
+          wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz
+          sudo tar -C /usr/local -xzf go1.16.3.linux-amd64.tar.gz
+          export PATH=$PATH:/usr/local/go/bin
+          git clone https://github.com/FiloSottile/mkcert && cd mkcert
+          go build -ldflags "-X main.Version=$(git describe --tags)"
+          sudo mv ./mkcert /usr/local/bin && cd ../
+          sudo chmod +x /usr/local/bin/mkcert
+          rm -Rf mkcert
+          curl -sL https://raw.githubusercontent.com/codeenigma/ce-dev/1.x/install.sh | /bin/sh -s -- linux
+
+      # Uses the ce-dev stack to run a test Drupal deploy
+      - name: Run a test ce-dev deploy with Drupal 9
+        run: |
+          ce-dev create -p test -t drupal9 -d ~/test
+          cd ~/test
+          ce-dev init
+          ce-dev start
+          ce-dev provision
+          ce-dev deploy
+          curl https://www.test.local
+        shell: bash
diff --git a/ce-dev/ansible/common.yml b/ce-dev/ansible/common.yml
index 98babe50..2cffc05e 100644
--- a/ce-dev/ansible/common.yml
+++ b/ce-dev/ansible/common.yml
@@ -1,5 +1,5 @@
 # Common vars for local development.
-is_local: yes
+is_local: true
 ce_deploy:
   username: "ce-dev"
   own_repository: "https://github.com/codeenigma/ce-deploy.git"
diff --git a/ce-dev/ansible/examples/drupal8/deploy.yml b/ce-dev/ansible/examples/drupal8/deploy.yml
index 87c6efad..7c8def57 100644
--- a/ce-dev/ansible/examples/drupal8/deploy.yml
+++ b/ce-dev/ansible/examples/drupal8/deploy.yml
@@ -37,14 +37,35 @@
   pre_tasks:
     # You can safely remove these steps once you have a working composer.json.
     - name: Download composer file.
-      get_url:
-        url: https://raw.githubusercontent.com/drupal/recommended-project/8.8.x/composer.json
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/drupal/recommended-project/8.9.x/composer.json
         dest: "{{ deploy_path }}/composer.json"
-        force: no
+        force: false
+    - name: Adding composer/installers plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.composer/installers true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding drupal/core-composer-scaffold plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.drupal/core-composer-scaffold true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding drupal/core-project-message plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.drupal/core-project-message true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true
+        working_dir: "{{ deploy_path }}"
     - name: Install drush.
-      command: 
-        cmd: composer require drush/drush
-        chdir: "{{ deploy_path }}"
+      community.general.composer:
+        command: require
+        arguments: drush/drush:10.*
+        working_dir: "{{ deploy_path }}"
   roles:
     - _init # Sets some variables the deploy scripts rely on.
     - composer # Composer install step.
diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml
index deeef9ee..b7579ac4 100644
--- a/ce-dev/ansible/examples/drupal9/deploy.yml
+++ b/ce-dev/ansible/examples/drupal9/deploy.yml
@@ -37,14 +37,15 @@
   pre_tasks:
     # You can safely remove these steps once you have a working composer.json.
     - name: Download composer file.
-      get_url:
-        url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/drupal/recommended-project/9.5.x/composer.json
         dest: "{{ deploy_path }}/composer.json"
-        force: no
+        force: false
     - name: Install drush.
-      command:
-        cmd: composer require drush/drush:11.*
-        chdir: "{{ deploy_path }}"
+      community.general.composer:
+        command: require
+        arguments: drush/drush:11.*
+        working_dir: "{{ deploy_path }}"
   roles:
     - _init # Sets some variables the deploy scripts rely on.
     - composer # Composer install step.
diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml
index 9844206b..09db8ead 100644
--- a/ce-dev/ansible/examples/localgov/deploy.yml
+++ b/ce-dev/ansible/examples/localgov/deploy.yml
@@ -37,18 +37,45 @@
   pre_tasks:
     # You can safely remove these steps once you have a working composer.json.
     - name: Download composer file.
-        get_url:
-          url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json
-          dest: "{{ deploy_path }}/composer.json"
-          force: false
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/drupal/recommended-project/9.1.x/composer.json
+        dest: "{{ deploy_path }}/composer.json"
+        force: false
+    - name: Adding composer/installers plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.composer/installers true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding drupal/core-composer-scaffold plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.drupal/core-composer-scaffold true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding drupal/core-project-message plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.drupal/core-project-message true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding dealerdirect/phpcodesniffer-composer-installer plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.dealerdirect/phpcodesniffer-composer-installer true
+        working_dir: "{{ deploy_path }}"
+    - name: Adding cweagans/composer-patches plugin to composer config.
+      community.general.composer:
+        command: config
+        arguments: allow-plugins.cweagans/composer-patches true
+        working_dir: "{{ deploy_path }}"
     - name: Install drush.
-        command:
-          cmd: composer require drush/drush:11.*
-          chdir: "{{ deploy_path }}"
+      community.general.composer:
+        command: require
+        arguments: drush/drush:10.*
+        working_dir: "{{ deploy_path }}"
     - name: Install localgov.
-        command:
-          cmd: composer require localgovdrupal/localgov
-          chdir: "{{ deploy_path }}"
+      community.general.composer:
+        command: require
+        arguments: localgovdrupal/localgov
+        working_dir: "{{ deploy_path }}"
   roles:
     - _init # Sets some variables the deploy scripts rely on.
     - composer # Composer install step.
@@ -56,4 +83,4 @@
     - config_generate # Generates settings.php
     #    - sync/database_sync # Grab database from a remote server.
     - database_apply # Run drush updb and config import.
-    - _exit # Some common housekeeping.
\ No newline at end of file
+    - _exit # Some common housekeeping.

From e30479273ea22c262ac353551f242412f1ee8391 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 21 Jul 2023 12:19:49 +0300
Subject: [PATCH 079/139] default_to_gzip (#338)

---
 .../templates/regular-backups.sh.j2                       | 8 ++++----
 .../database_backup-mysql/tasks/cleanup-dump.yml          | 2 +-
 .../database_backup-mysql/tasks/deploy-dump.yml           | 2 +-
 .../database_backup-mysql/tasks/revert-dump.yml           | 2 +-
 .../database_sync/database_sync-mysql/defaults/main.yml   | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
index 33c27a5a..691163f5 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2
@@ -6,20 +6,20 @@ DBHOST='{{ database.host }}'
 DB_NAME='{{ database.original.database }}'
 CURRENT_DBNAME='{{ database.name }}'
 TARGET_DIR="{{ cron_mysql_backup.dumps_directory }}/$DBHOST"
-TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2"
+TARBALL="$DB_NAME-$(date -Iseconds).sql.gz"
 KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1))
 
 backup(){
   mysqldump {{ cron_mysql_backup.mysqldump_params }} \
-	    -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL"
+	    -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | gzip > "$TARGET_DIR/$TARBALL"
   ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME"
 }
 
 cleanup(){
-  if [ "$(find "$TARGET_DIR" -name '*sql.bz2'  | wc -l)" -lt "$KEEP" ]; then
+  if [ "$(find "$TARGET_DIR" -name '*.sql.*'  | wc -l)" -lt "$KEEP" ]; then
     return 0
   fi
-  rm "$(find $TARGET_DIR -name '*.sql.bz2' | sort | head -n 1)"
+  rm "$(find $TARGET_DIR -name '*.sql.*' | sort | head -n 1)"
   cleanup
 }
 
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
index 2609cabe..734aad4d 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml
@@ -2,7 +2,7 @@
 # We assume it's safe to only go back 50 build back.
 - name: Delete mysql dumps.
   ansible.builtin.file:
-    path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2"
+    path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.*"
     state: absent
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 9f6f6959..9c11857f 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -23,7 +23,7 @@
   run_once: true
 
 - name: Take a database dump.
-  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | gzip > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz"
   args:
     executable: /bin/bash
   when: previous_build_number > 0
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 4fee5ceb..08059e30 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -1,7 +1,7 @@
 ---
 - name: Unpack dump file.
   ansible.builtin.unarchive:
-    src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2"
+    src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz"
     dest: "/tmp"
     remote_src: true
   run_once: true
diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 88143a7b..98ac19bf 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -2,7 +2,7 @@
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
   cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
-  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
+  archival_method: "gzip" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.

From a8c29c6f770a8f393174a62b93c24929f73ceda3 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 4 Aug 2023 17:08:19 +0300
Subject: [PATCH 080/139] update_for_database_names_handling (#340)

---
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 7e52bd07..25b5bfb8 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -157,14 +157,14 @@
 
 - name: Drop target database.
   ansible.builtin.command:
-    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'"
+    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists `{{ mysql_sync_target_database }}`;'"
 
 - name: Recreate target database.
   ansible.builtin.command:
-    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'"
+    cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database `{{ mysql_sync_target_database }}`;'"
 
 - name: Repopulate database from dump.
-  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}"
+  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} `{{ mysql_sync_target_database }}` < {{ mysql_sync_target_dump_unpacked_path }}"
   args:
     executable: /bin/bash
 

From 27d7588ef9cba3b28d9dba642aaf7a4a0c088755 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 4 Aug 2023 20:00:07 +0300
Subject: [PATCH 081/139] update_for_database_names_handling_fix (#342)

---
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 25b5bfb8..3f94e47d 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -164,7 +164,7 @@
     cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database `{{ mysql_sync_target_database }}`;'"
 
 - name: Repopulate database from dump.
-  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} `{{ mysql_sync_target_database }}` < {{ mysql_sync_target_dump_unpacked_path }}"
+  ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}"
   args:
     executable: /bin/bash
 

From 84488a82ebc10a246326283415418f6526cc5b3a Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 10 Aug 2023 17:10:35 +0200
Subject: [PATCH 082/139] Adding syncing for feature branches. (#346)

---
 docs/roles/_init.md                           |  8 ++++++
 .../sync/database_sync/database_sync-mysql.md |  2 +-
 roles/_init/README.md                         |  8 ++++++
 roles/_init/defaults/main.yml                 |  8 ++++++
 .../database_apply-drupal8/tasks/main.yml     | 26 +++++++++++++++++++
 .../database_sync-mysql/README.md             |  2 +-
 6 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index edf1f5d3..8118c9ea 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -34,9 +34,17 @@ drupal:
       cron:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
+      feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
+      # For syncing database and files on a feature branch initial build
+      mysql_sync: # see sync/database_sync for docs
+        databases: []
+      files_sync: # see sync/files_sync for docs
+        cleanup: true
+        directories: []
 mautic:
   image_path: "media/images"
   force_install: false
+# Used for custom build time tools like cachetool
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index d70d770e..c0c2954b 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -7,7 +7,7 @@ Sync MySQL databases between environments.
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
   cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
-  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
+  archival_method: "gzip" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.
diff --git a/roles/_init/README.md b/roles/_init/README.md
index edf1f5d3..8118c9ea 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -34,9 +34,17 @@ drupal:
       cron:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
+      feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
+      # For syncing database and files on a feature branch initial build
+      mysql_sync: # see sync/database_sync for docs
+        databases: []
+      files_sync: # see sync/files_sync for docs
+        cleanup: true
+        directories: []
 mautic:
   image_path: "media/images"
   force_install: false
+# Used for custom build time tools like cachetool
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 57ee7e44..2fad2e47 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -25,9 +25,17 @@ drupal:
       cron:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
+      feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
+      # For syncing database and files on a feature branch initial build
+      mysql_sync: # see sync/database_sync for docs
+        databases: []
+      files_sync: # see sync/files_sync for docs
+        cleanup: true
+        directories: []
 mautic:
   image_path: "media/images"
   force_install: false
+# Used for custom build time tools like cachetool
 bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index c423fad9..7faf684f 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -25,6 +25,32 @@
   when: (previous_build_number == 0) or (site.force_install is defined and site.force_install)
   register: _drush_output
 
+- name: Sync database.
+  ansible.builtin.import_role:
+    name: sync/database_sync
+  vars:
+    mysql_sync: "{{ site.mysql_sync }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  when:
+    - (previous_build_number == 0) or (site.force_install is defined and site.force_install)
+    - site.feature_branch | default(false)
+    - site.mysql_sync | length > 0
+
+- name: Sync files.
+  ansible.builtin.import_role:
+    name: sync/files_sync
+  vars:
+    files_sync: "{{ site.files_sync }}"
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  when:
+    - (previous_build_number == 0) or (site.force_install is defined and site.force_install)
+    - site.feature_branch | default(false)
+    - site.files_sync | length > 0
+
 - name: Show drush output.
   ansible.builtin.debug:
     msg: "{{ _drush_output }}"
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index d70d770e..c0c2954b 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -7,7 +7,7 @@ Sync MySQL databases between environments.
 mysql_sync:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here.
   cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes.
-  archival_method: "bzip2" # oprions are "bzip2" or "gzip".
+  archival_method: "gzip" # oprions are "bzip2" or "gzip".
   databases:
     - source:
         # Name of the database to take a dump from.

From 50232382aee7635afd50f1017f3199a4751d5475 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 10 Aug 2023 18:25:34 +0200
Subject: [PATCH 083/139] Feature branching pr 1.x (#348)

* Adding syncing for feature branches.

* Cannot loop over import_role, changing for include_role.
---
 docs/roles/database_backup/database_backup-mysql.md           | 4 ++--
 roles/database_apply/database_apply-drupal8/tasks/main.yml    | 4 ++--
 roles/database_backup/database_backup-mysql/README.md         | 4 ++--
 roles/database_backup/database_backup-mysql/defaults/main.yml | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index b52413bf..c52620f6 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -22,8 +22,8 @@ mysql_backup:
   # This is useful for locked-down setups where you do not have GRANT permissions.
   credentials_handling: rotate
   databases:
-    - database: "{{ project_name }}_{{ build_type }}"
-      user: "{{ project_name }}_{{ build_type }}"
+    - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
+      user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
 
 ```
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 7faf684f..7faa03c7 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -26,7 +26,7 @@
   register: _drush_output
 
 - name: Sync database.
-  ansible.builtin.import_role:
+  ansible.builtin.include_role:
     name: sync/database_sync
   vars:
     mysql_sync: "{{ site.mysql_sync }}"
@@ -39,7 +39,7 @@
     - site.mysql_sync | length > 0
 
 - name: Sync files.
-  ansible.builtin.import_role:
+  ansible.builtin.include_role:
     name: sync/files_sync
   vars:
     files_sync: "{{ site.files_sync }}"
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index b52413bf..c52620f6 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -22,8 +22,8 @@ mysql_backup:
   # This is useful for locked-down setups where you do not have GRANT permissions.
   credentials_handling: rotate
   databases:
-    - database: "{{ project_name }}_{{ build_type }}"
-      user: "{{ project_name }}_{{ build_type }}"
+    - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
+      user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
 
 ```
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index f7e070e9..1cbc2e11 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -17,6 +17,6 @@ mysql_backup:
   # This is useful for locked-down setups where you do not have GRANT permissions.
   credentials_handling: rotate
   databases:
-    - database: "{{ project_name }}_{{ build_type }}"
-      user: "{{ project_name }}_{{ build_type }}"
+    - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
+      user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"

From aea9803e5d52360c0f82feb6201324c6b90f398a Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 11 Aug 2023 13:33:06 +0200
Subject: [PATCH 084/139] Feature branching pr 1.x (#350)

* Adding syncing for feature branches.

* Cannot loop over import_role, changing for include_role.

* Adding ability to specify an exact filename for a settings template for Drupal 8 and above.
---
 docs/roles/_init.md                              | 16 ++++++++++------
 docs/roles/composer.md                           |  3 ++-
 docs/roles/config_generate.md                    |  2 +-
 .../config_generate/config_generate-drupal8.md   |  8 ++++++++
 docs/roles/npm.md                                |  3 ++-
 roles/_init/README.md                            | 16 ++++++++++------
 roles/_init/defaults/main.yml                    | 16 ++++++++++------
 roles/composer/README.md                         |  3 ++-
 roles/composer/defaults/main.yml                 |  4 ++--
 roles/config_generate/README.md                  |  2 +-
 .../config_generate-drupal8/README.md            |  8 ++++++++
 .../config_generate-drupal8/defaults/main.yml    |  4 ++++
 .../config_generate-drupal8/tasks/drush.yml      |  5 +++--
 .../config_generate-drupal8/tasks/settings.yml   |  5 +++--
 roles/config_generate/defaults/main.yml          |  2 +-
 roles/live_symlink/defaults/main.yml             |  4 ++--
 roles/npm/README.md                              |  3 ++-
 roles/npm/defaults/main.yml                      |  4 ++--
 18 files changed, 73 insertions(+), 35 deletions(-)
 create mode 100644 roles/config_generate/config_generate-drupal8/defaults/main.yml

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 8118c9ea..d561aac2 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -35,12 +35,16 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build
-      mysql_sync: # see sync/database_sync for docs
-        databases: []
-      files_sync: # see sync/files_sync for docs
-        cleanup: true
-        directories: []
+      # For syncing database and files on a feature branch initial build - include all variables if used
+      mysql_sync: {} # see sync/database_sync for docs
+      #  mysqldump_params: "{{ _mysqldump_params }}"
+      #  cleanup: true
+      #  archival_method: gzip
+      #  databases: []
+      files_sync: {} # see sync/files_sync for docs
+      #  unique_workspace: false
+      #  cleanup: true
+      #  directories: []
 mautic:
   image_path: "media/images"
   force_install: false
diff --git a/docs/roles/composer.md b/docs/roles/composer.md
index 17144004..74765170 100644
--- a/docs/roles/composer.md
+++ b/docs/roles/composer.md
@@ -14,7 +14,7 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -24,6 +24,7 @@ composer:
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
   templates: []
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/config_generate.md b/docs/roles/config_generate.md
index 5428dd8b..fe7aecfe 100644
--- a/docs/roles/config_generate.md
+++ b/docs/roles/config_generate.md
@@ -11,7 +11,7 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
diff --git a/docs/roles/config_generate/config_generate-drupal8.md b/docs/roles/config_generate/config_generate-drupal8.md
index 20e32b5b..75382886 100644
--- a/docs/roles/config_generate/config_generate-drupal8.md
+++ b/docs/roles/config_generate/config_generate-drupal8.md
@@ -4,4 +4,12 @@ Generates settings.php file for Drupal 8.
 <!--ENDTOC-->
 
 <!--ROLEVARS-->
+## Default variables
+```yaml
+config_generate_drupal:
+  # If you need to specify an exact file name for a template file to use for a build, you can do it here.
+  drupal_settings_file_name: settings.php
+  drush_settings_file_name: drush.yml
+```
+
 <!--ENDROLEVARS-->
diff --git a/docs/roles/npm.md b/docs/roles/npm.md
index 385e1c84..97f06cdc 100644
--- a/docs/roles/npm.md
+++ b/docs/roles/npm.md
@@ -18,7 +18,7 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -28,6 +28,7 @@ npm:
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
   templates: []
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 8118c9ea..d561aac2 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -35,12 +35,16 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build
-      mysql_sync: # see sync/database_sync for docs
-        databases: []
-      files_sync: # see sync/files_sync for docs
-        cleanup: true
-        directories: []
+      # For syncing database and files on a feature branch initial build - include all variables if used
+      mysql_sync: {} # see sync/database_sync for docs
+      #  mysqldump_params: "{{ _mysqldump_params }}"
+      #  cleanup: true
+      #  archival_method: gzip
+      #  databases: []
+      files_sync: {} # see sync/files_sync for docs
+      #  unique_workspace: false
+      #  cleanup: true
+      #  directories: []
 mautic:
   image_path: "media/images"
   force_install: false
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 2fad2e47..5fcee089 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -26,12 +26,16 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build
-      mysql_sync: # see sync/database_sync for docs
-        databases: []
-      files_sync: # see sync/files_sync for docs
-        cleanup: true
-        directories: []
+      # For syncing database and files on a feature branch initial build - include all variables if used
+      mysql_sync: {} # see sync/database_sync for docs
+      #  mysqldump_params: "{{ _mysqldump_params }}"
+      #  cleanup: true
+      #  archival_method: gzip
+      #  databases: []
+      files_sync: {} # see sync/files_sync for docs
+      #  unique_workspace: false
+      #  cleanup: true
+      #  directories: []
 mautic:
   image_path: "media/images"
   force_install: false
diff --git a/roles/composer/README.md b/roles/composer/README.md
index 17144004..74765170 100644
--- a/roles/composer/README.md
+++ b/roles/composer/README.md
@@ -14,7 +14,7 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -24,6 +24,7 @@ composer:
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
   templates: []
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/composer/defaults/main.yml b/roles/composer/defaults/main.yml
index 2e6eec4a..d0dfbd1d 100644
--- a/roles/composer/defaults/main.yml
+++ b/roles/composer/defaults/main.yml
@@ -7,7 +7,7 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -16,4 +16,4 @@ composer:
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
-  templates: []
\ No newline at end of file
+  templates: []
diff --git a/roles/config_generate/README.md b/roles/config_generate/README.md
index 5428dd8b..fe7aecfe 100644
--- a/roles/config_generate/README.md
+++ b/roles/config_generate/README.md
@@ -11,7 +11,7 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
diff --git a/roles/config_generate/config_generate-drupal8/README.md b/roles/config_generate/config_generate-drupal8/README.md
index 20e32b5b..75382886 100644
--- a/roles/config_generate/config_generate-drupal8/README.md
+++ b/roles/config_generate/config_generate-drupal8/README.md
@@ -4,4 +4,12 @@ Generates settings.php file for Drupal 8.
 <!--ENDTOC-->
 
 <!--ROLEVARS-->
+## Default variables
+```yaml
+config_generate_drupal:
+  # If you need to specify an exact file name for a template file to use for a build, you can do it here.
+  drupal_settings_file_name: settings.php
+  drush_settings_file_name: drush.yml
+```
+
 <!--ENDROLEVARS-->
diff --git a/roles/config_generate/config_generate-drupal8/defaults/main.yml b/roles/config_generate/config_generate-drupal8/defaults/main.yml
new file mode 100644
index 00000000..a6b7634d
--- /dev/null
+++ b/roles/config_generate/config_generate-drupal8/defaults/main.yml
@@ -0,0 +1,4 @@
+config_generate_drupal:
+  # If you need to specify an exact file name for a template file to use for a build, you can do it here.
+  drupal_settings_file_name: settings.php
+  drush_settings_file_name: drush.yml
\ No newline at end of file
diff --git a/roles/config_generate/config_generate-drupal8/tasks/drush.yml b/roles/config_generate/config_generate-drupal8/tasks/drush.yml
index f4b7ae7a..0bedd46d 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/drush.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/drush.yml
@@ -4,6 +4,7 @@
     src: '{{ item }}'
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml"
   with_first_found:
-    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml.j2"
-    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml"
+    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drush_settings_file_name }}.j2"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drush_settings_file_name }}"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ config_generate_drupal.drush_settings_file_name }}"
     - "drush.yml.j2"
\ No newline at end of file
diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index ebf182e8..6f698ba5 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -23,6 +23,7 @@
     src: "{{ item }}"
     dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php"
   with_first_found:
-    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2"
-    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php"
+    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drupal_settings_file_name }}.j2"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drupal_settings_file_name }}"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ config_generate_drupal.drupal_settings_file_name }}"
     - "settings.php.j2"
diff --git a/roles/config_generate/defaults/main.yml b/roles/config_generate/defaults/main.yml
index 5258730e..8f1987eb 100644
--- a/roles/config_generate/defaults/main.yml
+++ b/roles/config_generate/defaults/main.yml
@@ -3,7 +3,7 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
diff --git a/roles/live_symlink/defaults/main.yml b/roles/live_symlink/defaults/main.yml
index 4b9fb449..eca23ad6 100644
--- a/roles/live_symlink/defaults/main.yml
+++ b/roles/live_symlink/defaults/main.yml
@@ -6,7 +6,7 @@ live_symlink:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -15,4 +15,4 @@ live_symlink:
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
-  templates: []
\ No newline at end of file
+  templates: []
diff --git a/roles/npm/README.md b/roles/npm/README.md
index 385e1c84..97f06cdc 100644
--- a/roles/npm/README.md
+++ b/roles/npm/README.md
@@ -18,7 +18,7 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -28,6 +28,7 @@ npm:
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
   templates: []
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/npm/defaults/main.yml b/roles/npm/defaults/main.yml
index be6bdc3f..a29fcac7 100644
--- a/roles/npm/defaults/main.yml
+++ b/roles/npm/defaults/main.yml
@@ -11,7 +11,7 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: wether to create the target if it does not exists.
+  # create: whether to create the target if it does not exists.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #  - src: '/var/simplesaml/etc'
@@ -20,4 +20,4 @@ npm:
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
   # dest: can only be relative to the root of your repository (eg. 'www/config.php', 'var/mysettings.php')
-  templates: []
\ No newline at end of file
+  templates: []

From bff8a40f5030312f2702d7c147826e55e3a8cd32 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 22 Sep 2023 15:44:16 +0200
Subject: [PATCH 085/139] Improving db handling and fixing adding databases
 later. (#358)

* Improving db handling and fixing adding databases later.

* Bad spacing caught by linter.

* Making an explicit debug entry if we skip database population.

* Allowing override of 'handling' on a database-by-database basis.

* Tab spacing errors caught by linter.

* Syntax error in my debug.

* Being more verbose in task naming.

* Enforcing bash as the interpretor for shell.

* Adding new database handling to dump and none types.

* Accidentally doubled the args key on a task.

* Variable naming error.

* Add -w flag to grep to match only the whole database name when checking for db existence.
---
 .../database_backup/database_backup-mysql.md  |  1 +
 .../database_backup-mysql/README.md           |  1 +
 .../database_backup-mysql/defaults/main.yml   |  1 +
 .../tasks/deploy-dump.yml                     | 11 +++-
 .../tasks/deploy-none.yml                     | 11 +++-
 .../tasks/deploy-rolling.yml                  | 50 +++++++++++++------
 .../database_backup-mysql/tasks/deploy.yml    | 13 ++++-
 7 files changed, 70 insertions(+), 18 deletions(-)

diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index c52620f6..3a7ec471 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -25,6 +25,7 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
+      #handling: static # optional override to the main handling method on a per database basis
 
 ```
 
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index c52620f6..3a7ec471 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -25,6 +25,7 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
+      #handling: static # optional override to the main handling method on a per database basis
 
 ```
 
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 1cbc2e11..777cc6f9 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -20,3 +20,4 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
+      #handling: static # optional override to the main handling method on a per database basis
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 9c11857f..49bda27e 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -4,6 +4,15 @@
 - ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
+- name: Check if the database exists already.
+  ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}"
+  register: _build_db_status
+  failed_when: _build_db_status.rc > 1 # only exit on abnormal errors
+  run_once: true
+  args:
+    executable: /bin/bash
+
+# Create database if this is an initial build or it doesn't exist yet
 - name: Create initial database.
   community.mysql.mysql_db:
     name: "{{ _mysql_build_database_name }}"
@@ -11,7 +20,7 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when: previous_build_number == 0
+  when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true
 
 - name: Ensure the dump directory exists.
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index dea8fe9f..4c4b1624 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -4,6 +4,15 @@
 - ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
+- name: Check if the database exists already.
+  ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}"
+  register: _build_db_status
+  failed_when: _build_db_status.rc > 1 # only exit on abnormal errors
+  run_once: true
+  args:
+    executable: /bin/bash
+
+# Create database if this is an initial build or it doesn't exist yet
 - name: Create initial database.
   community.mysql.mysql_db:
     name: "{{ _mysql_build_database_name }}"
@@ -11,5 +20,5 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when: previous_build_number == 0
+  when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index aec45596..30fbaa62 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -4,26 +4,46 @@
     _mysql_build_database_name: "{{ database.database }}_{{ build_number }}"
 - ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}"
-  # Note: we don't use the mysql_db Ansible module on purpose.
-  # If database already exists, we want to fail and not override it
-  # with previous build.
-  # @TODO fix this so we check if the database exists and exit with
-  # the proper plugin instead of using command.
-- name: Create new database.
-  ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE `{{ _mysql_build_database_name }}`;"
+
+- name: Check if the new database exists already.
+  ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}"
+  register: _build_db_status
+  failed_when: _build_db_status.rc == 0 # we want the build to fail if the database exists
+  run_once: true
+  args:
+    executable: /bin/bash
+
+- name: Create a new database.
+  community.mysql.mysql_db:
+    name: "{{ _mysql_build_database_name }}"
+    state: present
+    config_file: "{{ database.credentials_file }}"
+    config_overrides_defaults: true
   run_once: true
 
-#- name: Create a new database.
-#  community.mysql.mysql_db:
-#    name: "{{ _mysql_build_database_name }}"
-#    state: present
-#    config_file: "{{ database.credentials_file }}"
-#    config_overrides_defaults: true
-#  run_once: true
+- name: Check if the previous database exists.
+  ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_previous_build_database_name }}"
+  register: _previous_db_status
+  failed_when: _previous_db_status.rc > 1 # only fail if we get an unexpected exitcode
+  run_once: true
+  args:
+    executable: /bin/bash
 
+# Importing with shell for speed, we cannot import with mysql_db unless we create a dump file first
 - name: Populate new database.
   ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}"
   args:
     executable: /bin/bash
-  when: previous_build_number > 0
+  when:
+    - previous_build_number > 0
+    - _previous_db_status.rc == 0 # only run if we have a database
+  run_once: true
+
+# Making it clear if we skipped the population of the new database
+- name: Populating database skipped.
+  ansible.builtin.debug:
+    msg: "### Attention - the new database was NOT populated!"
+  when:
+    - previous_build_number > 0
+    - _previous_db_status.rc != 0
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index 4160806c..4304c3e4 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -82,7 +82,18 @@
     _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}"
   when: mysql_backup.credentials_handling == 'rotate'
 
-- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml"
+- name: "Set database handling type to {{ mysql_backup.handling }}."
+  ansible.builtin.set_fact:
+    _mysql_handling: "{{ mysql_backup.handling }}"
+# If we have a specific instruction for handling this database differently, use it.
+- name: Override database handling type for this database, if specified.
+  ansible.builtin.set_fact:
+    _mysql_handling: "{{ database.handling }}"
+  when:
+    - database.handling is defined
+    - database.handling | length > 0
+- name: Execute backup tasks.
+  ansible.builtin.include_tasks: "deploy-{{ _mysql_handling }}.yml"
 
 # We append privileges instead of replacing,
 # to allow this role to be looped over,

From 0ed0378994858dfbc202f2e5ba4c501d24bba788 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 22 Sep 2023 17:54:55 +0200
Subject: [PATCH 086/139] Bug fixes pr 1.x (#366)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.
---
 .../cache_clear-opcache/defaults/main.yml         |  4 ++--
 .../database_apply-drupal8/tasks/main.yml         | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
index 222325ce..e851d795 100644
--- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
@@ -7,6 +7,6 @@ cache_clear_opcache:
   # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
-  clear_apcu: false
-  clear_stat: false
+  clear_apcu: true
+  clear_stat: true
   # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined
diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml
index 7faa03c7..1935ac79 100644
--- a/roles/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml
@@ -13,6 +13,21 @@
     - www_user != deploy_user
     - previous_build_number == 0
 
+- name: Fix file permissions for config directory.
+  ansible.builtin.file:
+    state: directory
+    path: "{{ deploy_path }}/{{ site.config_sync_directory }}"
+    owner: "{{ www_user }}"
+    group: "{{ www_user }}"
+    mode: '0775' # in this case often our deploy user is in the web user group and will need to be able to manipulate config
+  become: true
+  with_items: "{{ drupal.sites }}"
+  loop_control:
+    loop_var: site
+  when:
+    - www_user != deploy_user
+    - previous_build_number == 0
+
 - name: Install Drupal.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}"

From f8a7eae5da76477753d223edd6d87a6a4be72df8 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 25 Sep 2023 10:36:53 +0200
Subject: [PATCH 087/139] Bug fixes pr 1.x (#368)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.
---
 roles/cache_clear/cache_clear-opcache/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
index e851d795..fd75afb9 100644
--- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
@@ -7,6 +7,6 @@ cache_clear_opcache:
   # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
-  clear_apcu: true
+  clear_apcu: false
   clear_stat: true
   # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined

From df327b4a63f2237c752e7268af6962de81a1c1e4 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 25 Sep 2023 18:04:09 +0200
Subject: [PATCH 088/139] Bug fixes pr 1.x (#372)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.
---
 roles/deploy_code/tasks/cleanup.yml | 148 ++++++++++++++--------------
 1 file changed, 74 insertions(+), 74 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index b3a4f132..76c7e810 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -1,4 +1,18 @@
 ---
+- name: Build warning.
+  ansible.builtin.debug:
+    msg: "### WARNING - if you are using a server terminal exit NOW! You have 20 seconds to comply..."
+  run_once: true
+  when:
+    - deploy_code.mount_type == "squashfs"
+
+- name: Give people 20 seconds to exit terminal sessions.
+  ansible.builtin.pause:
+    seconds: 20
+  run_once: true
+  when:
+    - deploy_code.mount_type == "squashfs"
+
 - name: Ensure codebase is writable.
   ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi"
@@ -125,84 +139,70 @@
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
 
-- name: Check if we have a mount already.
-  ansible.builtin.shell:
-    cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}"
-  args:
-    executable: /bin/bash
-  failed_when: false
-  register: _deploy_code_mount_check
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-
-- name: Reload any services that might be keeping the loop device busy.
-  ansible.builtin.service:
-    name: "{{ www_service }}"
-    state: reloaded
-  with_items: "{{ deploy_code.services }}"
-  loop_control:
-    loop_var: www_service
-  become: true
-  when:
-    - deploy_code.service_action == "reload"
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check.rc == 0
-    - deploy_code.services | length > 0
-
-- name: Stop any services that might be keeping the loop device busy.
-  ansible.builtin.service:
-    name: "{{ www_service }}"
-    state: stopped
-  with_items: "{{ deploy_code.services }}"
-  loop_control:
-    loop_var: www_service
-  become: true
-  when:
-    - deploy_code.service_action == "stop"
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check.rc == 0
-    - deploy_code.services | length > 0
-
-- name: Unmount existing SquashFS image.
-  ansible.builtin.command:
-    cmd: "umount {{ deploy_base_path }}"
-  become: true
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check.rc == 0
-
-- name: Mount new SquashFS image.
-  ansible.builtin.command:
-    cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop"
-  become: true
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-
-- name: Start any services we stopped.
-  ansible.builtin.service:
-    name: "{{ www_service }}"
-    state: started
-  with_items: "{{ deploy_code.services }}"
-  loop_control:
-    loop_var: www_service
-  become: true
+- name: Unmount and remount squashfs images.
   when:
-    - deploy_code.service_action == "stop"
     - deploy_code.mount_sync is defined
     - deploy_code.mount_sync | length > 1
     - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_check is succeeded
-    - deploy_code.services | length > 0
+  block:
+    - name: Check if we have a mount already.
+      ansible.builtin.shell:
+        cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}"
+      args:
+        executable: /bin/bash
+      failed_when: false
+      register: _deploy_code_mount_check
+
+    - name: Reload any services that might be keeping the loop device busy.
+      ansible.builtin.service:
+        name: "{{ www_service }}"
+        state: reloaded
+      with_items: "{{ deploy_code.services }}"
+      loop_control:
+        loop_var: www_service
+      become: true
+      when:
+        - _deploy_code_mount_check.rc == 0
+        - deploy_code.service_action == "reload"
+        - deploy_code.services | length > 0
+
+    - name: Stop any services that might be keeping the loop device busy.
+      ansible.builtin.service:
+        name: "{{ www_service }}"
+        state: stopped
+      with_items: "{{ deploy_code.services }}"
+      loop_control:
+        loop_var: www_service
+      become: true
+      when:
+        - _deploy_code_mount_check.rc == 0
+        - deploy_code.service_action == "stop"
+        - deploy_code.services | length > 0
+
+    - name: Unmount existing SquashFS image.
+      ansible.builtin.command:
+        cmd: "umount {{ deploy_base_path }}"
+      become: true
+      when:
+        - _deploy_code_mount_check.rc == 0
+
+    - name: Mount new SquashFS image.
+      ansible.builtin.command:
+        cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop"
+      become: true
+
+    - name: Start any services we stopped.
+      ansible.builtin.service:
+        name: "{{ www_service }}"
+        state: started
+      with_items: "{{ deploy_code.services }}"
+      loop_control:
+        loop_var: www_service
+      become: true
+      when:
+        - _deploy_code_mount_check.rc == 0
+        - deploy_code.service_action == "stop"
+        - deploy_code.services | length > 0
 
 - name: Trigger an infrastructure rebuild.
   ansible.builtin.include_role:

From 7b775eea4ddd64e6d6360d80a5ba8d1871119327 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 27 Sep 2023 13:19:03 +0200
Subject: [PATCH 089/139] Bug fixes pr 1.x (#374)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.
---
 .../config_generate-drupal8/tasks/settings.yml                | 4 ++++
 .../config_generate-drupal8/templates/settings.php.j2         | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index 6f698ba5..4cbf8f5e 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -18,6 +18,10 @@
     mode: 0775
   become: "{{ false if www_user == deploy_user else true }}"
 
+- name: Generate or retrieve hash salt value.
+  ansible.builtin.set_fact:
+    _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name  + '_' + build_type + '/drupal/hash_salt', length=74, chars=['ascii_letters', 'digits']) }}"
+
 - name: Generates settings.php file.
   ansible.builtin.template:
     src: "{{ item }}"
diff --git a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2 b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2
index e5d43204..895f654f 100644
--- a/roles/config_generate/config_generate-drupal8/templates/settings.php.j2
+++ b/roles/config_generate/config_generate-drupal8/templates/settings.php.j2
@@ -23,3 +23,5 @@ $settings['file_public_path'] = '{{ build_public_file_path }}';
 $config_directories['sync'] = '{{ build_config_sync_directory }}';
 // Drupal 8.8
 $settings['config_sync_directory'] = '{{ build_config_sync_directory }}';
+
+$settings['hash_salt'] = '{{ _drupal_hash_salt }}';

From 00a636c3c7e0b1058080ecd5b6228af4c0a8578b Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 27 Sep 2023 13:56:07 +0200
Subject: [PATCH 090/139] Bug fixes pr 1.x (#375)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.
---
 .../config_generate/config_generate-drupal8/tasks/settings.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index 4cbf8f5e..79e0ad5e 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -20,7 +20,7 @@
 
 - name: Generate or retrieve hash salt value.
   ansible.builtin.set_fact:
-    _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name  + '_' + build_type + '/drupal/hash_salt', length=74, chars=['ascii_letters', 'digits']) }}"
+    _drupal_hash_salt: "{{ lookup('ansible.builtin.password', _ce_deploy_data_dir + '/' + project_name  + '_' + build_type + '/drupal/hash_salt length=74 chars=ascii_letters,digits') }}"
 
 - name: Generates settings.php file.
   ansible.builtin.template:

From b033cddcaf0745c350d82fcf8ae9240def443399 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matej=20=C5=A0tajduhar?=
 <30931414+matej5@users.noreply.github.com>
Date: Wed, 4 Oct 2023 18:05:03 +0200
Subject: [PATCH 091/139] r66858-adding-variable-to-skip-long-running-tasks
 (#352)

Co-authored-by: Matej Stajduhar <matej.stajduhar@codeenigma.com>
---
 roles/deploy_code/defaults/main.yml | 2 ++
 roles/deploy_code/tasks/cleanup.yml | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index f9b84739..7b68c3a7 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -35,6 +35,8 @@ deploy_code:
   service_action: reload
   # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
   rebuild_infra: false
+  # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default
+  fix_cleanup_perms: true
   # Details of API call to trigger. See api_call role.
   api_call:
     type: gitlab
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 76c7e810..776b0557 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -18,7 +18,9 @@
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi"
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
   become: true
-  when: "www_user != deploy_user"
+  when:
+    - "www_user != deploy_user"
+    - deploy_code.fix_cleanup_perms
 
 - name: Ensure permissions are set on deploy directory.
   ansible.builtin.shell:

From bcfa95a001e0e62ccdc066ddc326531cfffedf4c Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 4 Oct 2023 18:05:34 +0200
Subject: [PATCH 092/139] Fixing documentation for opcache role. (#354)

---
 roles/cache_clear/cache_clear-opcache/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
index fd75afb9..d7f4af62 100644
--- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml
+++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml
@@ -4,7 +4,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
+  # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false

From fb43059938aabedd111867898ff26f356195edb5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matej=20=C5=A0tajduhar?=
 <30931414+matej5@users.noreply.github.com>
Date: Wed, 4 Oct 2023 18:09:26 +0200
Subject: [PATCH 093/139] r67016-changing-path-for-database_backup-task (#356)

* r67016-changing-path-for-database_backup-task

* Slight refactor to use a directory we can safelist with rkhunter.

* Docs update.

* Removing rogue extra line.

---------

Co-authored-by: Matej Stajduhar <matej.stajduhar@codeenigma.com>
Co-authored-by: gregharvey <greg.harvey@gmail.com>
---
 docs/roles/cache_clear/cache_clear-opcache.md              | 2 +-
 docs/roles/database_backup/database_backup-mysql.md        | 3 +++
 roles/cache_clear/cache_clear-opcache/README.md            | 2 +-
 roles/database_backup/database_backup-mysql/README.md      | 3 +++
 .../database_backup-mysql/defaults/main.yml                | 2 ++
 .../database_backup/database_backup-mysql/tasks/deploy.yml | 7 ++++++-
 6 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md
index 465307e2..bb0bd05d 100644
--- a/docs/roles/cache_clear/cache_clear-opcache.md
+++ b/docs/roles/cache_clear/cache_clear-opcache.md
@@ -18,7 +18,7 @@ cache_clear_opcache:
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
-  clear_stat: false
+  clear_stat: true
   # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined
 
 ```
diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index 3a7ec471..6637e2da 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -8,6 +8,8 @@ mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
+  mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
@@ -27,6 +29,7 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: static # optional override to the main handling method on a per database basis
 
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md
index 465307e2..bb0bd05d 100644
--- a/roles/cache_clear/cache_clear-opcache/README.md
+++ b/roles/cache_clear/cache_clear-opcache/README.md
@@ -18,7 +18,7 @@ cache_clear_opcache:
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
-  clear_stat: false
+  clear_stat: true
   # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined
 
 ```
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index 3a7ec471..6637e2da 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -8,6 +8,8 @@ mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
+  mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
@@ -27,6 +29,7 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: static # optional override to the main handling method on a per database basis
 
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 777cc6f9..1091f8d7 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -3,6 +3,8 @@ mysql_backup:
   handling: rolling
   dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build"
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
+  # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
+  mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index 4304c3e4..8d93c118 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -71,6 +71,11 @@
     - _legacy_static_creds.stat.exists
 ### End of legacy handling.
 
+- name: Ensure the temporary password directory exists.
+  ansible.builtin.file:
+    state: directory
+    path: "{{ mysql_backup.mysql_password_path }}"
+
 - ansible.builtin.set_fact:
     _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}"
   when: mysql_backup.credentials_handling == 'static'
@@ -79,7 +84,7 @@
     _mysql_build_user_name: "{{ database.user }}_{{ build_number }}"
   when: mysql_backup.credentials_handling == 'rotate'
 - ansible.builtin.set_fact:
-    _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}"
+    _mysql_build_password: "{{ lookup('password', '{{ mysql_backup.mysql_password_path }}') }}"
   when: mysql_backup.credentials_handling == 'rotate'
 
 - name: "Set database handling type to {{ mysql_backup.handling }}."

From a4c04f6ab8afde8e2fe99fbdc3cb8bf58340fe60 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 31 Oct 2023 17:44:05 +0100
Subject: [PATCH 094/139] 67412 callback plugin fail no hosts pr 1.x (#377)

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Removing obsolete SOPS plugin and adding new callback to fail when no hosts match.
---
 plugins/callback/fail_on_no_hosts.py |  30 +++++
 plugins/vars/sops_vars.py            | 163 ---------------------------
 2 files changed, 30 insertions(+), 163 deletions(-)
 create mode 100644 plugins/callback/fail_on_no_hosts.py
 delete mode 100644 plugins/vars/sops_vars.py

diff --git a/plugins/callback/fail_on_no_hosts.py b/plugins/callback/fail_on_no_hosts.py
new file mode 100644
index 00000000..862ae9a4
--- /dev/null
+++ b/plugins/callback/fail_on_no_hosts.py
@@ -0,0 +1,30 @@
+import sys
+
+from ansible.plugins.callback import CallbackBase
+
+DOCUMENTATION = '''
+name: fail_on_no_hosts
+callback_type: aggregate
+requirements:
+    - enable in configuration
+short_description: Exits with code 1 if no play hosts are matched
+version_added: "2.0"
+description:
+    - This callback overrides the default 'v2_playbook_on_no_hosts_matched' method with one that exits instead of just notifying.
+'''
+
+class CallbackModule(CallbackBase):
+    """
+    This callback module exists non-zero if no hosts match
+    """
+    CALLBACK_VERSION = 2.0
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'fail_on_no_hosts'
+    CALLBACK_NEEDS_WHITELIST = False
+
+    def __init__(self):
+        super(CallbackModule, self).__init__()
+
+    def v2_playbook_on_no_hosts_matched(self):
+        self._display.display("failed: no hosts matched", color=C.COLOR_ERROR)
+        sys.exit(1)
diff --git a/plugins/vars/sops_vars.py b/plugins/vars/sops_vars.py
deleted file mode 100644
index eca3f3b9..00000000
--- a/plugins/vars/sops_vars.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright: (c) 2019, Arduino, srl
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-#
-#############################################
-
-from __future__ import (absolute_import, division, print_function)
-__metaclass__ = type
-
-DOCUMENTATION = '''
-    vars: sops_vars
-    author: Edoardo Tenani (@endorama) <e.tenani@arduino.cc>
-    version_added: "2.10"
-    short_description: Loading sops-encrypted vars files
-    description:
-        - Load encrypted YAML files into correspondind groups/hosts in group_vars/ and host_vars/ directories.
-        - Files are encrypted prior to reading, making this plugin an effective companion to host_group_vars plugin.
-        - Files are restricted to .sops.yaml, .sops.yml, .sops.json extensions.
-        - Hidden files are ignored.
-    options:
-      _valid_extensions:
-        default: [".sops.yml", ".sops.yaml", ".sops.json"]
-        description:
-          - "Check all of these extensions when looking for 'variable' files which should be YAML or JSON or vaulted versions of these."
-          - 'This affects vars_files, include_vars, inventory and vars plugins among others.'
-        type: list
-'''
-
-import os
-from ansible import constants as C
-from ansible.errors import AnsibleParserError
-from ansible.module_utils._text import to_bytes, to_native, to_text
-from ansible.plugins.vars import BaseVarsPlugin
-from ansible.inventory.host import Host
-from ansible.inventory.group import Group
-from ansible.utils.vars import combine_vars
-from ansible.errors import AnsibleError
-from subprocess import Popen, PIPE
-from ansible.utils.display import Display
-display = Display()
-
-FOUND = {}
-DEFAULT_VALID_EXTENSIONS = [".sops.yaml", ".sops.yml", ".sops.json"]
-
-# From https://github.com/mozilla/sops/blob/master/cmd/sops/codes/codes.go
-# Should be manually updated
-sops_error_codes = {
-    1: "SopsErrorGeneric",
-    2: "SopsCouldNotReadInputFile",
-    3: "SopsCouldNotWriteOutputFile",
-    4: "SopsErrorDumpingTree",
-    5: "SopsErrorReadingConfig",
-    6: "SopsErrorInvalidKMSEncryptionContextFormat",
-    7: "SopsErrorInvalidSetFormat",
-    8: "SopsErrorConflictingParameters",
-    21: "SopsErrorEncryptingMac",
-    23: "SopsErrorEncryptingTree",
-    24: "SopsErrorDecryptingMac",
-    25: "SopsErrorDecryptingTree",
-    49: "SopsCannotChangeKeysFromNonExistentFile",
-    51: "SopsMacMismatch",
-    52: "SopsMacNotFound",
-    61: "SopsConfigFileNotFound",
-    85: "SopsKeyboardInterrupt",
-    91: "SopsInvalidTreePathFormat",
-    100: "SopsNoFileSpecified",
-    128: "SopsCouldNotRetrieveKey",
-    111: "SopsNoEncryptionKeyFound",
-    200: "SopsFileHasNotBeenModified",
-    201: "SopsNoEditorFound",
-    202: "SopsFailedToCompareVersions",
-    203: "SopsFileAlreadyEncrypted"
-}
-
-
-class SopsError(AnsibleError):
-    ''' extend AnsibleError class with sops specific informations '''
-
-    def __init__(self, filename, exit_code, message,):
-        exception_name = sops_error_codes[exit_code]
-        message = "error with file %s: %s exited with code %d: %s" % (filename, exception_name, exit_code, message)
-        super(SopsError, self).__init__(message=message)
-
-
-def decrypt_with_sops(filename):
-    display.vvvv(u"sops --decrypt %s" % filename)
-
-    # Run sops directly as python module is deprecated
-    process = Popen(["sops", "--decrypt", filename], stdout=PIPE, stderr=PIPE)
-    (output, err) = process.communicate()
-    exit_code = process.wait()
-
-    # DO NOT display output
-    # is the decrypted secret and would easily end in logs :)
-    # if output:
-    #     display.vvvv(output)
-
-    # sops logs always to stderr ( stdout is used for file content )
-    if err:
-        display.vvvv(err)
-
-    if exit_code > 0:
-        if exit_code in sops_error_codes.keys():
-            raise SopsError(filename, exit_code, err)
-        else:
-            raise AnsibleError(message=err)
-
-    return output
-
-
-class VarsModule(BaseVarsPlugin):
-
-    def get_vars(self, loader, path, entities, cache=True):
-        ''' parses the inventory file '''
-
-        if not isinstance(entities, list):
-            entities = [entities]
-
-        super(VarsModule, self).get_vars(loader, path, entities)
-
-        data = {}
-        for entity in entities:
-            if isinstance(entity, Host):
-                subdir = 'host_vars'
-            elif isinstance(entity, Group):
-                subdir = 'group_vars'
-            else:
-                raise AnsibleParserError("Supplied entity must be Host or Group, got %s instead" % (type(entity)))
-
-            # avoid 'chroot' type inventory hostnames /path/to/chroot
-            if not entity.name.startswith(os.path.sep):
-                try:
-                    found_files = []
-                    # load vars
-                    b_opath = os.path.realpath(to_bytes(os.path.join(self._basedir, subdir)))
-                    opath = to_text(b_opath)
-                    key = '%s.%s' % (entity.name, opath)
-                    self._display.vvvv("key: %s" % (key))
-                    if cache and key in FOUND:
-                        found_files = FOUND[key]
-                    else:
-                        # no need to do much if path does not exist for basedir
-                        if os.path.exists(b_opath):
-                            if os.path.isdir(b_opath):
-                                self._display.debug("\tprocessing dir %s" % opath)
-                                found_files = loader.find_vars_files(opath, entity.name)
-                                found_files = [file_path for file_path in found_files
-                                               if any(file_path.endswith(extension) for extension in DEFAULT_VALID_EXTENSIONS)]
-                                FOUND[key] = found_files
-                            else:
-                                self._display.warning("Found %s that is not a directory, skipping: %s" % (subdir, opath))
-
-                    for found in found_files:
-                        file_content = decrypt_with_sops(found)
-                        new_data = loader.load(file_content)
-                        if new_data:  # ignore empty files
-                            data = combine_vars(data, new_data)
-
-                except Exception as e:
-                    raise AnsibleParserError(to_native(e))
-
-        return data

From d651f84c11e9a89b40e807e63e5c4454a8e0a597 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 31 Oct 2023 18:28:24 +0100
Subject: [PATCH 095/139] Bug fixes pr 1.x (#379)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.
---
 plugins/callback/fail_on_no_hosts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/callback/fail_on_no_hosts.py b/plugins/callback/fail_on_no_hosts.py
index 862ae9a4..e022c0a4 100644
--- a/plugins/callback/fail_on_no_hosts.py
+++ b/plugins/callback/fail_on_no_hosts.py
@@ -1,5 +1,6 @@
 import sys
 
+from ansible import constants as C
 from ansible.plugins.callback import CallbackBase
 
 DOCUMENTATION = '''

From 6c9213a679f098107b4d4e645ebf826eb963cf94 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 2 Nov 2023 13:02:47 +0100
Subject: [PATCH 096/139] Bug fixes pr 1.x (#381)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.
---
 plugins/README.md         |  16 ++++
 plugins/vars/sops_vars.py | 163 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 179 insertions(+)
 create mode 100644 plugins/README.md
 create mode 100644 plugins/vars/sops_vars.py

diff --git a/plugins/README.md b/plugins/README.md
new file mode 100644
index 00000000..58e31070
--- /dev/null
+++ b/plugins/README.md
@@ -0,0 +1,16 @@
+# Ansible Plugins
+This directory contains extra plugins for Ansible.
+
+## Vars plugins
+Here we include a plugin for handling SOPS decryption.
+
+## Callback plugins
+Here we have a small custom override that fails builds if there are no matching hosts found, to avoid `ce-deploy` incrementing the track file when it didn't actually run because of a host issue.
+
+## Enabling plugins
+To use these plugins you need to find the `# set plugin path directories here` section of `ansible.cfg` which should be kept in your `ce-deploy-config` repository. Add the paths to the plugin directories to enable the plugins, e.g.
+
+```
+vars_plugins     = /home/deploy/ce-deploy/plugins/vars:/usr/share/ansible/plugins/vars
+callback_plugins = /home/deploy/ce-deploy/plugins/callback:/usr/share/ansible/plugins/callback
+```
diff --git a/plugins/vars/sops_vars.py b/plugins/vars/sops_vars.py
new file mode 100644
index 00000000..9193d61c
--- /dev/null
+++ b/plugins/vars/sops_vars.py
@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2019, Arduino, srl
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+#############################################
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = '''
+    vars: sops_vars
+    author: Edoardo Tenani (@endorama) <e.tenani@arduino.cc>
+    version_added: "2.10"
+    short_description: Loading sops-encrypted vars files
+    description:
+        - Load encrypted YAML files into correspondind groups/hosts in group_vars/ and host_vars/ directories.
+        - Files are encrypted prior to reading, making this plugin an effective companion to host_group_vars plugin.
+        - Files are restricted to .sops.yaml, .sops.yml, .sops.json extensions.
+        - Hidden files are ignored.
+    options:
+      _valid_extensions:
+        default: [".sops.yml", ".sops.yaml", ".sops.json"]
+        description:
+          - "Check all of these extensions when looking for 'variable' files which should be YAML or JSON or vaulted versions of these."
+          - 'This affects vars_files, include_vars, inventory and vars plugins among others.'
+        type: list
+'''
+
+import os
+from ansible import constants as C
+from ansible.errors import AnsibleParserError
+from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.plugins.vars import BaseVarsPlugin
+from ansible.inventory.host import Host
+from ansible.inventory.group import Group
+from ansible.utils.vars import combine_vars
+from ansible.errors import AnsibleError
+from subprocess import Popen, PIPE
+from ansible.utils.display import Display
+display = Display()
+
+FOUND = {}
+DEFAULT_VALID_EXTENSIONS = [".sops.yaml", ".sops.yml", ".sops.json"]
+
+# From https://github.com/mozilla/sops/blob/master/cmd/sops/codes/codes.go
+# Should be manually updated
+sops_error_codes = {
+    1: "SopsErrorGeneric",
+    2: "SopsCouldNotReadInputFile",
+    3: "SopsCouldNotWriteOutputFile",
+    4: "SopsErrorDumpingTree",
+    5: "SopsErrorReadingConfig",
+    6: "SopsErrorInvalidKMSEncryptionContextFormat",
+    7: "SopsErrorInvalidSetFormat",
+    8: "SopsErrorConflictingParameters",
+    21: "SopsErrorEncryptingMac",
+    23: "SopsErrorEncryptingTree",
+    24: "SopsErrorDecryptingMac",
+    25: "SopsErrorDecryptingTree",
+    49: "SopsCannotChangeKeysFromNonExistentFile",
+    51: "SopsMacMismatch",
+    52: "SopsMacNotFound",
+    61: "SopsConfigFileNotFound",
+    85: "SopsKeyboardInterrupt",
+    91: "SopsInvalidTreePathFormat",
+    100: "SopsNoFileSpecified",
+    128: "SopsCouldNotRetrieveKey",
+    111: "SopsNoEncryptionKeyFound",
+    200: "SopsFileHasNotBeenModified",
+    201: "SopsNoEditorFound",
+    202: "SopsFailedToCompareVersions",
+    203: "SopsFileAlreadyEncrypted"
+}
+
+
+class SopsError(AnsibleError):
+    ''' extend AnsibleError class with sops specific informations '''
+
+    def __init__(self, filename, exit_code, message,):
+        exception_name = sops_error_codes[exit_code]
+        message = "error with file %s: %s exited with code %d: %s" % (filename, exception_name, exit_code, message)
+        super(SopsError, self).__init__(message=message)
+
+
+def decrypt_with_sops(filename):
+    display.vvvv(u"sops --decrypt %s" % filename)
+
+    # Run sops directly as python module is deprecated
+    process = Popen(["sops", "--decrypt", filename], stdout=PIPE, stderr=PIPE)
+    (output, err) = process.communicate()
+    exit_code = process.wait()
+
+    # DO NOT display output
+    # is the decrypted secret and would easily end in logs :)
+    # if output:
+    #     display.vvvv(output)
+
+    # sops logs always to stderr ( stdout is used for file content )
+    if err:
+        display.vvvv(err)
+
+    if exit_code > 0:
+        if exit_code in sops_error_codes.keys():
+            raise SopsError(filename, exit_code, err)
+        else:
+            raise AnsibleError(message=err)
+
+    return output
+
+
+class VarsModule(BaseVarsPlugin):
+
+    def get_vars(self, loader, path, entities, cache=True):
+        ''' parses the inventory file '''
+
+        if not isinstance(entities, list):
+            entities = [entities]
+
+        super(VarsModule, self).get_vars(loader, path, entities)
+
+        data = {}
+        for entity in entities:
+            if isinstance(entity, Host):
+                subdir = 'host_vars'
+            elif isinstance(entity, Group):
+                subdir = 'group_vars'
+            else:
+                raise AnsibleParserError("Supplied entity must be Host or Group, got %s instead" % (type(entity)))
+
+            # avoid 'chroot' type inventory hostnames /path/to/chroot
+            if not entity.name.startswith(os.path.sep):
+                try:
+                    found_files = []
+                    # load vars
+                    b_opath = os.path.realpath(to_bytes(os.path.join(self._basedir, subdir)))
+                    opath = to_text(b_opath)
+                    key = '%s.%s' % (entity.name, opath)
+                    self._display.vvvv("key: %s" % (key))
+                    if cache and key in FOUND:
+                        found_files = FOUND[key]
+                    else:
+                        # no need to do much if path does not exist for basedir
+                        if os.path.exists(b_opath):
+                            if os.path.isdir(b_opath):
+                                self._display.debug("\tprocessing dir %s" % opath)
+                                found_files = loader.find_vars_files(opath, entity.name)
+                                found_files = [file_path for file_path in found_files
+                                               if any(file_path.endswith(extension) for extension in DEFAULT_VALID_EXTENSIONS)]
+                                FOUND[key] = found_files
+                            else:
+                                self._display.warning("Found %s that is not a directory, skipping: %s" % (subdir, opath))
+
+                    for found in found_files:
+                        file_content = decrypt_with_sops(found)
+                        new_data = loader.load(file_content)
+                        if new_data:  # ignore empty files
+                            data = combine_vars(data, new_data)
+
+                except Exception as e:
+                    raise AnsibleParserError(to_native(e))
+
+        return data
\ No newline at end of file

From 7729a2e20b3ebd8dfbaa720b88c388d2c5b7da24 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Tue, 7 Nov 2023 17:10:21 +0100
Subject: [PATCH 097/139] Bug fixes pr 1.x (#383)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.
---
 roles/_init/tasks/drupal7.yml                             | 8 ++++++--
 roles/_init/tasks/drupal8.yml                             | 6 +++++-
 .../maintenance_mode-drupal-core/tasks/offline.yml        | 4 ++--
 .../maintenance_mode-drupal-core/tasks/online.yml         | 8 ++++----
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml
index 133b3d73..b1781c1b 100644
--- a/roles/_init/tasks/drupal7.yml
+++ b/roles/_init/tasks/drupal7.yml
@@ -1,8 +1,12 @@
 ---
-- name: Define Drush path.
+- name: Define path to drush for this build.
   set_fact:
     drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}"
 
-- name: Ensure we have Drush binary.
+- name: Define path to drush for currently live build. # usually the same as drush_bin for Drupal 7
+  set_fact:
+    drush_live_bin: "{{ drush_live_bin | default('{{ bin_directory }}/drush.phar') }}"
+
+- name: Ensure we have a drush binary.
   import_role:
     name: cli/drush
diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml
index ebe63b82..3792c489 100644
--- a/roles/_init/tasks/drupal8.yml
+++ b/roles/_init/tasks/drupal8.yml
@@ -1,4 +1,8 @@
 ---
-- name: Define Drush path.
+- name: Define path to drush for this build.
   set_fact:
     drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}"
+
+- name: Define path to drush for currently live build.
+  set_fact:
+    drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}"
\ No newline at end of file
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index 2d6a9310..78314024 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -1,7 +1,7 @@
 ---
 - name: Enable maintenance mode.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   args:
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
@@ -13,7 +13,7 @@
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Enable maintenance mode D7.
   ansible.builtin.shell:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 1"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 6b127f89..2598481a 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -1,9 +1,9 @@
 ---
 - name: Disable maintenance mode.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   args:
-    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:
@@ -12,8 +12,8 @@
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Disable maintenance mode D7.
   ansible.builtin.shell:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
-    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 0"
+    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:

From 3132c690da3356aca59826fca54bcc165af7c37f Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 9 Nov 2023 10:32:49 +0100
Subject: [PATCH 098/139] Bug fixes pr 1.x (#386)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.
---
 .../maintenance_mode-drupal-core/tasks/online.yml         | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 2598481a..6b127f89 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -1,9 +1,9 @@
 ---
 - name: Disable maintenance mode.
   ansible.builtin.command:
-    cmd: "{{ drush_live_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   args:
-    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:
@@ -12,8 +12,8 @@
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Disable maintenance mode D7.
   ansible.builtin.shell:
-    cmd: "{{ drush_live_bin }} -l {{ site.folder }} vset maintenance_mode 0"
-    chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:

From 5c44c3c386ec530d4faf7dbb107b9119fa4c16f5 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 13 Nov 2023 17:23:00 +0100
Subject: [PATCH 099/139] Bug fixes pr 1.x (#388)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.
---
 scripts/_common.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/_common.sh b/scripts/_common.sh
index 74ba9a60..04e14c18 100755
--- a/scripts/_common.sh
+++ b/scripts/_common.sh
@@ -21,19 +21,27 @@ BUILD_WORKSPACE_BASE="$OWN_DIR/build"
 DRY_RUN="no"
 VERBOSE="no"
 BOTO_PROFILE=""
+# Ensure build workspace exists.
 if [ ! -d "$BUILD_WORKSPACE_BASE" ]; then
     mkdir "$BUILD_WORKSPACE_BASE"
 fi
 BUILD_TMP_DIR=$(mktemp -d -p "$BUILD_WORKSPACE_BASE")
+# Ensure ce-deploy data directory exists.
 ANSIBLE_DATA_DIR="$OWN_DIR/data"
 if [ ! -d "$ANSIBLE_DATA_DIR" ]; then
     mkdir "$ANSIBLE_DATA_DIR"
 fi
+# Ensure directory for build track files exists.
 BUILD_TRACK_DIR="$OWN_DIR/track"
 if [ ! -d "$BUILD_TRACK_DIR" ]; then
   mkdir "$BUILD_TRACK_DIR"
 fi
 ANSIBLE_LOCATION=$(command -v ansible)
+# Load the contents of profile.d in case we added items to $PATH there.
+for f in /etc/profile.d/*; do
+# shellcheck source=/dev/null
+   . "$f"
+done
 # Parse options arguments.
 parse_options(){
   while [ "${1:-}" ]; do

From 66dee820db94c5734e3fd31811895596eda5229e Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 16 Nov 2023 17:33:49 +0100
Subject: [PATCH 100/139] Bug fixes pr 1.x (#391)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.
---
 roles/deploy_container/defaults/main.yml       |  2 +-
 .../cache_clear-drupal8/tasks/main.yml         | 16 ++--------------
 .../database_apply-drupal8/tasks/main.yml      | 18 +++---------------
 3 files changed, 6 insertions(+), 30 deletions(-)

diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index be178aca..f01dcb15 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -68,7 +68,7 @@ deploy_container:
     launch_type: FARGATE
     network_mode: awsvpc
     #volumes: [] # list of additional volumes to attach
-    target_group_name: example # 32 character limit
+    target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen
     target_group_protocol: http
     target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
index 431366cf..0f116adc 100644
--- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,16 +1,8 @@
 ---
-# Drupal 8 ships drush with the website code so we need the previous build in the path.
-- name: Stash the drush_bin variable.
-  ansible.builtin.set_fact:
-    _drush_bin_deploy: "{{ drush_bin }}"
-
-- name: Update location of drush for the sync cache clear command.
-  ansible.builtin.set_fact:
-    drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush"
-
+# Drupal 8 ships drush with the website code so to use the drush_live_bin variable
 - name: Clear Drupal cache.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y cr"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -18,7 +10,3 @@
   loop_control:
     loop_var: site
   run_once: true
-
-- name: Restore the drush_bin variable.
-  ansible.builtin.set_fact:
-    drush_bin: "{{ _drush_bin_deploy }}"
diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
index d13b0129..297e476b 100644
--- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
+++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml
@@ -1,16 +1,8 @@
 ---
-# Drupal 8 ships drush with the website code so we need the previous build in the path.
-- name: Stash the drush_bin variable.
-  ansible.builtin.set_fact:
-    _drush_bin_deploy: "{{ drush_bin }}"
-
-- name: Update location of drush for the update/config import commands.
-  ansible.builtin.set_fact:
-    drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush"
-
+# Drupal 8 ships drush with the website code so to use the drush_live_bin variable
 - name: Apply Drupal database updates.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y updb"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -27,7 +19,7 @@
 # This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run.
 - name: Import configuration.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
+    cmd: "{{ drush_live_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}"
     chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
@@ -47,7 +39,3 @@
 - name: Clear the cache.
   ansible.builtin.include_role:
     name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}"
-
-- name: Restore the drush_bin variable.
-  ansible.builtin.set_fact:
-    drush_bin: "{{ _drush_bin_deploy }}"

From 58f9526f0d2376e72ca7e20d998cfff8ac73e70d Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 24 Nov 2023 17:08:47 +0100
Subject: [PATCH 101/139] Bug fixes pr 1.x (#393)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.
---
 .../database_backup-mysql/tasks/revert-dump.yml   | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 08059e30..6040c73a 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -1,9 +1,16 @@
 ---
-- name: Unpack dump file.
-  ansible.builtin.unarchive:
-    src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz"
-    dest: "/tmp"
+- name: Move dump file to local disk.
+  ansible.builtin.copy:
     remote_src: true
+    src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.gz"
+    dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz"
+  run_once: true
+  when: previous_build_number > 0
+
+# unarchive module does not support archives not in tar.gz format
+- name: Unpack dump file.
+  ansible.builtin.command:
+    cmd: "gunzip /tmp/{{ database.database }}-{{ previous_build_number }}.sql.gz"
   run_once: true
   when: previous_build_number > 0
 

From 0b5d0603aacb0cc9122ac13d0592c14a3be47a29 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 21 Dec 2023 10:54:54 +0100
Subject: [PATCH 102/139] Ecs deployments pr 1.x (#397)

* Adding container push and build code.

* Fixing some minor issues.

* Docs updates and template flexibility improvement.

* Creating separate registry name for handling docker push.

* Trying to get the registry name right.

* Adding AWS stack components to ECS builds.

* Force a docker logout.

* Need to delegate docker commands to the controller.

* Trying deleting the credentials file instead.

* Fixing var name in subnet.yml.

* Missing some ACM variables.

* Allowing us to set wait timeout for TG creation.

* Uncommenting cpu and memory vars for ECS with Fargate.

* Missed a variable in ELB creation.

* Couple of linting fixes.

* Force building and tagging of containers as an option.

* Improving docs for deploy_container role.

* Providing container description example for ECS.

* Adding 'force' option to docker builds.

* Revert "Adding 'force' option to docker builds."

This reverts commit 5574faa182f023c20ec34ae6dc7125c92b4c7dcb.

* Adding options to force ECS component refreshes.

* Allowing users to toggle public IP and execute command mode.

* The ALB needs to be on public subnets, not the same private ones as the cluster.

* Updating documentation.

* Updating ECS docs.

* New deploy_container destroy action.

* Documentation update.

* Adding a note to docs about the container destroy action.
---
 docs/roles/cache_clear/cache_clear-opcache.md |   2 +-
 .../database_backup/database_backup-mysql.md  |   1 -
 docs/roles/deploy_code.md                     |   2 +
 docs/roles/deploy_container.md                |   5 +-
 .../cache_clear/cache_clear-opcache/README.md |   2 +-
 .../database_backup-mysql/README.md           |   1 -
 roles/deploy_code/README.md                   |   2 +
 roles/deploy_container/README.md              |   5 +-
 roles/deploy_container/defaults/main.yml      |   1 +
 .../deploy_container/tasks/action-cleanup.yml |   3 +
 .../deploy_container/tasks/action-create.yml  | 363 +++++++++++++++++
 .../deploy_container/tasks/action-destroy.yml | 254 ++++++++++++
 roles/deploy_container/tasks/main.yml         | 371 +-----------------
 13 files changed, 644 insertions(+), 368 deletions(-)
 create mode 100644 roles/deploy_container/tasks/action-cleanup.yml
 create mode 100644 roles/deploy_container/tasks/action-create.yml
 create mode 100644 roles/deploy_container/tasks/action-destroy.yml

diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md
index bb0bd05d..267485f4 100644
--- a/docs/roles/cache_clear/cache_clear-opcache.md
+++ b/docs/roles/cache_clear/cache_clear-opcache.md
@@ -14,7 +14,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
+  # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index 6637e2da..1200f97f 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -29,7 +29,6 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: static # optional override to the main handling method on a per database basis
 
-
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index 377ca80c..5d4cc722 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -121,6 +121,8 @@ deploy_code:
   service_action: reload
   # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
   rebuild_infra: false
+  # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default
+  fix_cleanup_perms: true
   # Details of API call to trigger. See api_call role.
   api_call:
     type: gitlab
diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index f127bb61..06847722 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -8,6 +8,8 @@ sudo usermod -aG docker deploy
 
 This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
 
+If you set the `deploy_container.action` to `destroy` then the role will also take care of tidying up containers. If it is an AWS ECS deployment then it will also tidy up the ECS service for you.
+
 ## AWS IAM requirements
 AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached.
 
@@ -41,6 +43,7 @@ It is worth noting that even if you put your containers on private subnets and c
 ```yaml
 ---
 deploy_container:
+  action: create # can also be destroy
   container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
@@ -109,7 +112,7 @@ deploy_container:
     launch_type: FARGATE
     network_mode: awsvpc
     #volumes: [] # list of additional volumes to attach
-    target_group_name: example # 32 character limit
+    target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen
     target_group_protocol: http
     target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md
index bb0bd05d..267485f4 100644
--- a/roles/cache_clear/cache_clear-opcache/README.md
+++ b/roles/cache_clear/cache_clear-opcache/README.md
@@ -14,7 +14,7 @@ cache_clear_opcache:
   # eg.
   # --fcgi=127.0.0.1:9000
   # Leave blank to use /etc/cachetool.yml
-  # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
+  # adapter: "--fcgi=127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version.
   # Bins to clear.
   clear_opcache: true
   clear_apcu: false
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index 6637e2da..1200f97f 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -29,7 +29,6 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: static # optional override to the main handling method on a per database basis
 
-
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index 377ca80c..5d4cc722 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -121,6 +121,8 @@ deploy_code:
   service_action: reload
   # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI.
   rebuild_infra: false
+  # Used to skip tasks to fix ownership and permissions, drupal needs this set to true by default
+  fix_cleanup_perms: true
   # Details of API call to trigger. See api_call role.
   api_call:
     type: gitlab
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index f127bb61..06847722 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -8,6 +8,8 @@ sudo usermod -aG docker deploy
 
 This can be handled automatically by [`ce-provision`](https://github.com/codeenigma/ce-provision) using the `ce_deploy` and `docker_ce` roles.
 
+If you set the `deploy_container.action` to `destroy` then the role will also take care of tidying up containers. If it is an AWS ECS deployment then it will also tidy up the ECS service for you.
+
 ## AWS IAM requirements
 AWS integration requires the AWS CLI user provided for `ce-deploy` to have certain managed AWS policies attached.
 
@@ -41,6 +43,7 @@ It is worth noting that even if you put your containers on private subnets and c
 ```yaml
 ---
 deploy_container:
+  action: create # can also be destroy
   container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
@@ -109,7 +112,7 @@ deploy_container:
     launch_type: FARGATE
     network_mode: awsvpc
     #volumes: [] # list of additional volumes to attach
-    target_group_name: example # 32 character limit
+    target_group_name: example # can have a maximum of 32 characters, must contain only alphanumeric characters or hyphens, and must not begin or end with a hyphen
     target_group_protocol: http
     target_group_port: 8080 # ports lower than 1024 will require the app to be configured to run as a privileged user in the Dockerfile
     target_group_wait_timeout: 200 # how long to wait for target group events to complete
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index f01dcb15..ff99d2b9 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -1,5 +1,6 @@
 ---
 deploy_container:
+  action: create # can also be destroy
   container_name: example-container
   container_tag: latest # tag will take format container_name:container_tag
   container_force_build: true # force Docker to build and tag a new image
diff --git a/roles/deploy_container/tasks/action-cleanup.yml b/roles/deploy_container/tasks/action-cleanup.yml
new file mode 100644
index 00000000..57312dbe
--- /dev/null
+++ b/roles/deploy_container/tasks/action-cleanup.yml
@@ -0,0 +1,3 @@
+---
+# @TODO we need container image cleanup here and potentially ECS Task Definition cleanup too.
+# Possibly useful link: https://stackoverflow.com/a/40949364
diff --git a/roles/deploy_container/tasks/action-create.yml b/roles/deploy_container/tasks/action-create.yml
new file mode 100644
index 00000000..cc06163d
--- /dev/null
+++ b/roles/deploy_container/tasks/action-create.yml
@@ -0,0 +1,363 @@
+---
+# Build and ship a container image
+- name: Create Dockerfile from template.
+  ansible.builtin.template:
+    src: "{{ deploy_container.dockerfile_template }}"
+    dest: "{{ deploy_container.docker_build_dir }}/Dockerfile"
+  delegate_to: localhost
+
+- name: Set Docker registry username and password.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "{{ deploy_container.docker_registry_user }}"
+    _docker_registry_password: "{{ deploy_container.docker_registry_pass }}"
+  delegate_to: localhost
+
+- name: Fetch AWS ECR registry login token. # token valid for 12 hours
+  ansible.builtin.command:
+    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+  register: _docker_registry_ecr_token
+
+- name: Set AWS ECR registry password.
+  ansible.builtin.set_fact:
+    _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Set AWS ECR registry username.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "AWS"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Remove Docker credentials file.
+  ansible.builtin.file:
+    state: absent
+    path: "/home/{{ deploy_user }}/.docker/config.json"
+  delegate_to: localhost
+
+- name: Log into Docker registry.
+  community.docker.docker_login:
+    registry_url: "{{ deploy_container.docker_registry_url }}"
+    username: "{{ _docker_registry_username }}"
+    password: "{{ _docker_registry_password }}"
+    reauthorize: true
+  delegate_to: localhost
+
+- name: Build and push container image.
+  community.docker.docker_image:
+    build:
+      path: "{{ deploy_container.docker_build_dir }}"
+    name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}"
+    tag: "{{ deploy_container.container_tag | default('latest') }}"
+    push: true
+    source: build
+    force_source: "{{ deploy_container.container_force_build }}"
+    force_tag: "{{ deploy_container.container_force_build }}"
+  delegate_to: localhost
+
+# Fetch the ACM role from ce-provision
+- name: Ensure the aws_acm directory exists.
+  ansible.builtin.file:
+    path: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}"
+    state: directory
+    mode: '0755'
+  delegate_to: localhost
+  with_items:
+    - tasks
+    - defaults
+
+- name: Fetch the aws_acm files.
+  ansible.builtin.get_url:
+    url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml"
+    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml"
+  delegate_to: localhost
+  with_items:
+    - tasks
+    - defaults
+
+- name: Fetch the aws_acm tasks.
+  ansible.builtin.get_url:
+    url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml
+    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml"
+  delegate_to: localhost
+
+# Gather all network information
+- name: Gather VPC information.
+  amazon.aws.ec2_vpc_net_info:
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    region: "{{ deploy_container.aws_ecs.region }}"
+    filters:
+      "tag:Name": "{{ deploy_container.aws_ecs.vpc_name }}"
+  register: _aws_ecs_cluster_vpc
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.vpc_name is defined
+    - deploy_container.aws_ecs.vpc_name | length > 0
+
+- name: Set the VPC id from name.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_vpc_id: "{{ _aws_ecs_cluster_vpc.vpcs[0].vpc_id }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.vpc_name is defined
+    - deploy_container.aws_ecs.vpc_name | length > 0
+
+- name: Use provided VPC id.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_vpc_id: "{{ deploy_container.aws_ecs.vpc_id }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - (deploy_container.aws_ecs.vpc_name is not defined or deploy_container.aws_ecs.vpc_name | length < 0)
+
+- name: Reset subnets lists.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_public_subnets_ids: []
+    _aws_ecs_cluster_private_subnets_ids: []
+  when: deploy_container.aws_ecs.enabled
+
+- name: Construct list of public subnet IDs.
+  ansible.builtin.include_tasks: subnet-public.yml
+  with_items: "{{ deploy_container.aws_ecs.elb_subnets }}"
+  loop_control:
+    loop_var: subnet
+  when: deploy_container.aws_ecs.enabled
+
+- name: Construct list of private subnet IDs.
+  ansible.builtin.include_tasks: subnet-private.yml
+  with_items: "{{ deploy_container.aws_ecs.service_subnets }}"
+  loop_control:
+    loop_var: subnet
+  when: deploy_container.aws_ecs.enabled
+
+# Construct AWS supporting assets
+- name: Create task definition.
+  community.aws.ecs_taskdefinition:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    family: "{{ deploy_container.aws_ecs.family_name }}"
+    execution_role_arn: "{{ deploy_container.aws_ecs.task_execution_role_arn }}"
+    task_role_arn: "{{ deploy_container.aws_ecs.task_role_arn | default(omit) }}"
+    containers: "{{ deploy_container.aws_ecs.containers }}"
+    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
+    cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}"
+    memory: "{{ deploy_container.aws_ecs.memory | default(omit) }}"
+    state: present
+    network_mode: "{{ deploy_container.aws_ecs.network_mode }}"
+    volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}"
+    force_create: "{{ deploy_container.aws_ecs.task_definition_force_create }}"
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create a target group with IP address targets.
+  community.aws.elb_target_group:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    protocol: "{{ deploy_container.aws_ecs.target_group_protocol }}"
+    port: "{{ deploy_container.aws_ecs.target_group_port }}"
+    vpc_id: "{{ _aws_ecs_cluster_vpc_id }}"
+    health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}"
+    health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}"
+    successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}"
+    target_type: ip
+    targets: "{{ deploy_container.aws_ecs.targets }}"
+    state: present
+    wait_timeout: "{{ deploy_container.aws_ecs.target_group_wait_timeout }}"
+    wait: true
+  register: _aws_ecs_target_group
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create SSL certificate for load balancer.
+  ansible.builtin.include_role:
+    name: aws_acm
+  vars:
+    aws_acm:
+      region: "{{ deploy_container.aws_ecs.region }}"
+      aws_profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+      tags: "{{ deploy_container.aws_ecs.tags }}"
+      export: false
+      domain_name: "{{ deploy_container.aws_ecs.domain_name }}"
+      extra_domains: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+      route_53:
+        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
+        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
+  when:
+    - deploy_container.aws_ecs.acm.create_cert
+    - deploy_container.aws_ecs.enabled
+
+- name: Default to provided SSL certificate ARN.
+  ansible.builtin.set_fact:
+    _ssl_certificate_ARN: "{{ deploy_container.aws_ecs.ssl_certificate_ARN }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: If provided, override SSL certificate ARN with the one received from ACM.
+  ansible.builtin.set_fact:
+    _ssl_certificate_ARN: "{{ aws_acm_certificate_arn }}"
+  when:
+    - deploy_container.aws_ecs.acm.create_cert
+    - deploy_container.aws_ecs.enabled
+
+- name: Define default ALB listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners_http:
+      Protocol: HTTP
+      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
+      DefaultActions:
+        - Type: forward
+          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
+      Rules: "{{ deploy_container.aws_ecs.elb_listener_http_rules }}"
+    _aws_ecs_cluster_listeners_redirect:
+      Protocol: HTTP
+      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
+      DefaultActions:
+        - Type: redirect
+          RedirectConfig:
+            Protocol: HTTPS
+            Host: "#{host}"
+            Query: "#{query}"
+            Path: "/#{path}"
+            Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
+            StatusCode: HTTP_301
+    _aws_ecs_cluster_listeners_https:
+      Protocol: HTTPS
+      Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
+      SslPolicy: "{{ deploy_container.aws_ecs.elb_ssl_policy }}"
+      Certificates:
+        - CertificateArn: "{{ _ssl_certificate_ARN }}"
+      DefaultActions:
+        - Type: forward
+          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
+      Rules: "{{ deploy_container.aws_ecs.elb_listener_https_rules }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Add HTTP listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_http ] }}"
+  when:
+    - _ssl_certificate_ARN | length < 1
+    - deploy_container.aws_ecs.enabled
+
+- name: Add HTTPS Listener.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_redirect, _aws_ecs_cluster_listeners_https ] }}"
+  when:
+    - _ssl_certificate_ARN | length > 1
+    - deploy_container.aws_ecs.enabled
+
+- name: Add custom Listeners.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_listeners: "{{ _aws_ecs_cluster_listeners + deploy_container.aws_ecs.elb_listeners }}"
+  when:
+    - deploy_container.aws_ecs.elb_listeners is defined
+    - deploy_container.aws_ecs.elb_listeners | length
+    - deploy_container.aws_ecs.enabled
+
+- name: Create an ALB.
+  amazon.aws.elb_application_lb:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    state: present
+    tags: "{{ deploy_container.aws_ecs.tags }}"
+    subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
+    security_groups: "{{ deploy_container.aws_ecs.elb_security_groups }}"
+    listeners: "{{ _aws_ecs_cluster_listeners }}"
+    idle_timeout: "{{ deploy_container.aws_ecs.elb_idle_timeout }}"
+    ip_address_type: "{{ deploy_container.aws_ecs.elb_ip_address_type }}"
+  register: _aws_ecs_cluster_alb
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Set task definition name.
+  ansible.builtin.set_fact:
+    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Set task definition revision if applicable.
+  ansible.builtin.set_fact:
+    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}:{{ deploy_container.aws_ecs.task_definition_revision }}"
+  when:
+    - deploy_container.aws_ecs.task_definition_revision | length > 0
+    - deploy_container.aws_ecs.enabled
+
+- name: Create ECS service.
+  community.aws.ecs_service:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: present
+    name: "{{ deploy_container.aws_ecs.family_name }}"
+    cluster: "{{ deploy_container.aws_ecs.cluster_name }}"
+    task_definition: "{{ _aws_ecs_service_task_definition }}"
+    desired_count: "{{ deploy_container.aws_ecs.task_count }}"
+    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
+    platform_version: LATEST
+    load_balancers: # see https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_LoadBalancer.html
+      - containerName: "{{ deploy_container.container_name }}"
+        containerPort: "{{ deploy_container.aws_ecs.target_group_port }}"
+        targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}"
+    network_configuration:
+      subnets: "{{ _aws_ecs_cluster_private_subnets_ids }}" # internal private subnet
+      security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
+      assign_public_ip: "{{ deploy_container.aws_ecs.service_public_container_ip }}"
+    tags: "{{ deploy_container.aws_ecs.tags }}"
+    enable_execute_command: "{{ deploy_container.aws_ecs.service_enable_ssm }}"
+    force_new_deployment: "{{ deploy_container.aws_ecs.service_force_refresh }}"
+    wait: true
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Create target tracking scaling policy for ECS service.
+  community.aws.application_autoscaling_policy:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: present
+    policy_name: "{{ deploy_container.aws_ecs.family_name }}"
+    service_namespace: ecs
+    resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}"
+    scalable_dimension: ecs:service:DesiredCount
+    minimum_tasks: "{{ deploy_container.aws_ecs.task_minimum_count }}"
+    maximum_tasks: "{{ deploy_container.aws_ecs.task_maximum_count }}"
+    policy_type: TargetTrackingScaling
+    target_tracking_scaling_policy_configuration:
+      PredefinedMetricSpecification:
+        PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}"
+      ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}"
+      ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}"
+      DisableScaleIn: false
+      TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}"
+  delegate_to: localhost
+  when: deploy_container.aws_ecs.enabled
+
+- name: Initialise the domains loop var with main domain entry DNS settings.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains:
+      - domain: "{{ deploy_container.aws_ecs.domain_name }}"
+        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
+        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
+  when: deploy_container.aws_ecs.enabled
+
+- name: Add extra_domains so we can loop through DNS records.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}"
+  loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+  when:
+    - deploy_container.aws_ecs.acm.extra_domains | length > 0
+    - deploy_container.aws_ecs.enabled
+
+- name: Add DNS records in Route 53.
+  amazon.aws.route53:
+    state: present
+    profile: "{{ item.aws_profile }}"
+    zone: "{{ item.zone }}"
+    record: "{{ item.domain }}"
+    type: CNAME
+    value: "{{ _aws_ecs_cluster_alb.dns_name }}"
+    overwrite: true
+  loop: "{{ _aws_ecs_cluster_dns_all_domains }}"
+  when:
+    - deploy_container.aws_ecs.route_53.zone | length > 0
+    - deploy_container.aws_ecs.enabled
diff --git a/roles/deploy_container/tasks/action-destroy.yml b/roles/deploy_container/tasks/action-destroy.yml
new file mode 100644
index 00000000..b0b6bd99
--- /dev/null
+++ b/roles/deploy_container/tasks/action-destroy.yml
@@ -0,0 +1,254 @@
+---
+# Be sure to include your deploy_container variables in your playbook
+- name: Set Docker registry username and password.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "{{ deploy_container.docker_registry_user }}"
+    _docker_registry_password: "{{ deploy_container.docker_registry_pass }}"
+  delegate_to: localhost
+
+- name: Fetch AWS ECR registry login token. # token valid for 12 hours
+  ansible.builtin.command:
+    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+  register: _docker_registry_ecr_token
+
+- name: Set AWS ECR registry password.
+  ansible.builtin.set_fact:
+    _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Set AWS ECR registry username.
+  ansible.builtin.set_fact:
+    _docker_registry_username: "AWS"
+  when: deploy_container.aws_ecr.enabled
+  delegate_to: localhost
+
+- name: Remove Docker credentials file.
+  ansible.builtin.file:
+    state: absent
+    path: "/home/{{ deploy_user }}/.docker/config.json"
+  delegate_to: localhost
+
+- name: Log into Docker registry.
+  community.docker.docker_login:
+    registry_url: "{{ deploy_container.docker_registry_url }}"
+    username: "{{ _docker_registry_username }}"
+    password: "{{ _docker_registry_password }}"
+    reauthorize: true
+  delegate_to: localhost
+
+- name: Destroy matching container images.
+  community.docker.docker_image:
+    name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}"
+    tag: "{{ deploy_container.container_tag | default('latest') }}"
+    force_absent: true
+    state: absent
+  delegate_to: localhost
+
+# Destroy AWS services
+- name: Get minimal ALB information before we destroy it.
+  amazon.aws.elb_application_lb_info:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    names: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    #include_attributes: false # @TODO - these attributes added in amazon.aws 7.0.0
+    #include_listeners: false
+    #include_listener_rules: false
+  register: _aws_ecs_cluster_alb
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Destroy ALB.
+  amazon.aws.elb_application_lb:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    state: absent
+    wait: true
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Destroy target group.
+  community.aws.elb_target_group:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
+    state: absent
+    wait: true
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Reduce task count to zero on ECS service.
+  community.aws.ecs_service:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: present
+    name: "{{ deploy_container.aws_ecs.family_name }}"
+    cluster: "{{ deploy_container.aws_ecs.cluster_name }}"
+    desired_count: 0
+    force_new_deployment: true
+    wait: true
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Destroy ECS service.
+  community.aws.ecs_service:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: absent
+    name: "{{ deploy_container.aws_ecs.family_name }}"
+    cluster: "{{ deploy_container.aws_ecs.cluster_name }}"
+    wait: true
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+# @TODO: We cannot currently use the module for this
+# See feature request: https://github.com/ansible-collections/community.aws/issues/2023
+#- name: Get task definition details.
+#  community.aws.ecs_taskdefinition_info:
+#    region: "{{ deploy_container.aws_ecs.region }}"
+#    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+#    task_definition: "{{ deploy_container.aws_ecs.family_name }}"
+#  register: _task_definition_info
+#  delegate_to: localhost
+
+- name: Ensure the task definitions ARN list variable is empty.
+  ansible.builtin.set_fact:
+    _task_definition_arns_list: []
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Get active task definition details.
+  ansible.builtin.command:
+    cmd: "aws ecs list-task-definitions --status ACTIVE --family-prefix {{ deploy_container.aws_ecs.family_name }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}"
+  register: _task_definition_arns_raw
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Convert CLI output of active ARNs to a YAML variable.
+  ansible.builtin.set_fact:
+    _task_definition_arns: "{{ _task_definition_arns_raw.stdout | from_json }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Create a clean list of task definition ARNs.
+  ansible.builtin.set_fact:
+    _task_definition_arns_list: "{{ _task_definition_arns.taskDefinitionArns }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Get inactive task definition details.
+  ansible.builtin.command:
+    cmd: "aws ecs list-task-definitions --status INACTIVE --family-prefix {{ deploy_container.aws_ecs.family_name }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}"
+  register: _task_definition_arns_raw
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Convert CLI output of inactive ARNs to a YAML variable.
+  ansible.builtin.set_fact:
+    _task_definition_arns: "{{ _task_definition_arns_raw.stdout | from_json }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Add inactive task definition ARNs to the YAML list.
+  ansible.builtin.set_fact:
+    _task_definition_arns_list: "{{ _task_definition_arns_list + _task_definition_arns.taskDefinitionArns }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Deregister task definitions.
+  community.aws.ecs_taskdefinition:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    arn: "{{ item }}"
+    containers: "{{ deploy_container.aws_ecs.containers }}"
+    state: absent
+  delegate_to: localhost
+  with_items: "{{ _task_definition_arns_list }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Delete task definitions.
+  ansible.builtin.command:
+    cmd: "aws ecs delete-task-definitions --task-definitions {{ item }} --region {{ deploy_container.aws_ecs.region }} --profile {{ deploy_container.aws_ecs.aws_profile }}"
+  delegate_to: localhost
+  with_items: "{{ _task_definition_arns_list }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Destroy scaling policy for ECS service.
+  community.aws.application_autoscaling_policy:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    state: absent
+    policy_name: "{{ deploy_container.aws_ecs.family_name }}"
+    service_namespace: ecs
+    policy_type: TargetTrackingScaling
+    resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}"
+    scalable_dimension: ecs:service:DesiredCount
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+
+# Clean up SSL certificates
+- name: Delete the main ACM certificate.
+  community.aws.acm_certificate:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    domain_name: "{{ deploy_container.aws_ecs.domain_name }}"
+    state: absent
+  delegate_to: localhost
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.acm.create_cert
+
+- name: Delete any extra ACM certificates.
+  community.aws.acm_certificate:
+    region: "{{ deploy_container.aws_ecs.region }}"
+    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
+    domain_name: "{{ item.domain }}"
+    state: absent
+  with_items: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+  when:
+    - deploy_container.aws_ecs.acm.extra_domains | length > 0
+    - deploy_container.aws_ecs.enabled
+  delegate_to: localhost
+
+# Clean up DNS
+- name: Initialise the domains loop var with main domain entry DNS settings.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains:
+      - domain: "{{ deploy_container.aws_ecs.domain_name }}"
+        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
+        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+
+- name: Add extra_domains so we can loop through DNS records.
+  ansible.builtin.set_fact:
+    _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}"
+  loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.acm.extra_domains | length > 0
+
+- name: Remove DNS records in Route 53.
+  amazon.aws.route53:
+    state: absent
+    profile: "{{ item.aws_profile }}"
+    zone: "{{ item.zone }}"
+    record: "{{ item.domain }}"
+    type: CNAME
+    value: "{{ _aws_ecs_cluster_alb.load_balancers[0].dns_name }}"
+  loop: "{{ _aws_ecs_cluster_dns_all_domains }}"
+  when:
+    - deploy_container.aws_ecs.enabled
+    - deploy_container.aws_ecs.route_53.zone | length > 0
diff --git a/roles/deploy_container/tasks/main.yml b/roles/deploy_container/tasks/main.yml
index 6c68bebd..ce3e618f 100644
--- a/roles/deploy_container/tasks/main.yml
+++ b/roles/deploy_container/tasks/main.yml
@@ -1,363 +1,10 @@
 ---
-# Build and ship a container image
-- name: Create Dockerfile from template.
-  ansible.builtin.template:
-    src: "{{ deploy_container.dockerfile_template }}"
-    dest: "{{ deploy_container.docker_build_dir }}/Dockerfile"
-  delegate_to: localhost
-
-- name: Set Docker registry username and password.
-  ansible.builtin.set_fact:
-    _docker_registry_username: "{{ deploy_container.docker_registry_user }}"
-    _docker_registry_password: "{{ deploy_container.docker_registry_pass }}"
-  delegate_to: localhost
-
-- name: Fetch AWS ECR registry login token. # token valid for 12 hours
-  ansible.builtin.command:
-    cmd: "aws ecr get-login-password --region {{ deploy_container.aws_ecr.region }} --profile {{ deploy_container.aws_ecr.aws_profile }}"
-  when: deploy_container.aws_ecr.enabled
-  delegate_to: localhost
-  register: _docker_registry_ecr_token
-
-- name: Set AWS ECR registry password.
-  ansible.builtin.set_fact:
-    _docker_registry_password: "{{ _docker_registry_ecr_token.stdout }}"
-  when: deploy_container.aws_ecr.enabled
-  delegate_to: localhost
-
-- name: Set AWS ECR registry username.
-  ansible.builtin.set_fact:
-    _docker_registry_username: "AWS"
-  when: deploy_container.aws_ecr.enabled
-  delegate_to: localhost
-
-- name: Remove Docker credentials file.
-  ansible.builtin.file:
-    state: absent
-    path: "/home/{{ deploy_user }}/.docker/config.json"
-  delegate_to: localhost
-
-- name: Log into Docker registry.
-  community.docker.docker_login:
-    registry_url: "{{ deploy_container.docker_registry_url }}"
-    username: "{{ _docker_registry_username }}"
-    password: "{{ _docker_registry_password }}"
-    reauthorize: true
-  delegate_to: localhost
-
-- name: Build and push container image.
-  community.docker.docker_image:
-    build:
-      path: "{{ deploy_container.docker_build_dir }}"
-    name: "{{ deploy_container.docker_registry_name }}/{{ deploy_container.container_name }}"
-    tag: "{{ deploy_container.container_tag | default('latest') }}"
-    push: true
-    source: build
-    force_source: "{{ deploy_container.container_force_build }}"
-    force_tag: "{{ deploy_container.container_force_build }}"
-  delegate_to: localhost
-
-# Fetch the ACM role from ce-provision
-- name: Ensure the aws_acm directory exists.
-  ansible.builtin.file:
-    path: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}"
-    state: directory
-    mode: '0755'
-  delegate_to: localhost
-  with_items:
-    - tasks
-    - defaults
-
-- name: Fetch the aws_acm files.
-  ansible.builtin.get_url:
-    url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml"
-    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml"
-  delegate_to: localhost
-  with_items:
-    - tasks
-    - defaults
-
-- name: Fetch the aws_acm tasks.
-  ansible.builtin.get_url:
-    url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml
-    dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml"
-  delegate_to: localhost
-
-# Gather all network information
-- name: Gather VPC information.
-  amazon.aws.ec2_vpc_net_info:
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    region: "{{ deploy_container.aws_ecs.region }}"
-    filters:
-      "tag:Name": "{{ deploy_container.aws_ecs.vpc_name }}"
-  register: _aws_ecs_cluster_vpc
-  delegate_to: localhost
-  when:
-    - deploy_container.aws_ecs.enabled
-    - deploy_container.aws_ecs.vpc_name is defined
-    - deploy_container.aws_ecs.vpc_name | length > 0
-
-- name: Set the VPC id from name.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_vpc_id: "{{ _aws_ecs_cluster_vpc.vpcs[0].vpc_id }}"
-  when:
-    - deploy_container.aws_ecs.enabled
-    - deploy_container.aws_ecs.vpc_name is defined
-    - deploy_container.aws_ecs.vpc_name | length > 0
-
-- name: Use provided VPC id.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_vpc_id: "{{ deploy_container.aws_ecs.vpc_id }}"
-  when:
-    - deploy_container.aws_ecs.enabled
-    - (deploy_container.aws_ecs.vpc_name is not defined or deploy_container.aws_ecs.vpc_name | length < 0)
-
-- name: Reset subnets lists.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_public_subnets_ids: []
-    _aws_ecs_cluster_private_subnets_ids: []
-  when: deploy_container.aws_ecs.enabled
-
-- name: Construct list of public subnet IDs.
-  ansible.builtin.include_tasks: subnet-public.yml
-  with_items: "{{ deploy_container.aws_ecs.elb_subnets }}"
-  loop_control:
-    loop_var: subnet
-  when: deploy_container.aws_ecs.enabled
-
-- name: Construct list of private subnet IDs.
-  ansible.builtin.include_tasks: subnet-private.yml
-  with_items: "{{ deploy_container.aws_ecs.service_subnets }}"
-  loop_control:
-    loop_var: subnet
-  when: deploy_container.aws_ecs.enabled
-
-# Construct AWS supporting assets
-- name: Create task definition.
-  community.aws.ecs_taskdefinition:
-    region: "{{ deploy_container.aws_ecs.region }}"
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    family: "{{ deploy_container.aws_ecs.family_name }}"
-    execution_role_arn: "{{ deploy_container.aws_ecs.task_execution_role_arn }}"
-    task_role_arn: "{{ deploy_container.aws_ecs.task_role_arn | default(omit) }}"
-    containers: "{{ deploy_container.aws_ecs.containers }}"
-    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
-    cpu: "{{ deploy_container.aws_ecs.cpu | default(omit) }}"
-    memory: "{{ deploy_container.aws_ecs.memory | default(omit) }}"
-    state: present
-    network_mode: "{{ deploy_container.aws_ecs.network_mode }}"
-    volumes: "{{ deploy_container.aws_ecs.volumes | default(omit) }}"
-    force_create: "{{ deploy_container.aws_ecs.task_definition_force_create }}"
-  delegate_to: localhost
-  when: deploy_container.aws_ecs.enabled
-
-- name: Create a target group with IP address targets.
-  community.aws.elb_target_group:
-    region: "{{ deploy_container.aws_ecs.region }}"
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
-    protocol: "{{ deploy_container.aws_ecs.target_group_protocol }}"
-    port: "{{ deploy_container.aws_ecs.target_group_port }}"
-    vpc_id: "{{ _aws_ecs_cluster_vpc_id }}"
-    health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}"
-    health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}"
-    successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}"
-    target_type: ip
-    targets: "{{ deploy_container.aws_ecs.targets }}"
-    state: present
-    wait_timeout: "{{ deploy_container.aws_ecs.target_group_wait_timeout }}"
-    wait: true
-  register: _aws_ecs_target_group
-  delegate_to: localhost
-  when: deploy_container.aws_ecs.enabled
-
-- name: Create SSL certificate for load balancer.
-  ansible.builtin.include_role:
-    name: aws_acm
-  vars:
-    aws_acm:
-      region: "{{ deploy_container.aws_ecs.region }}"
-      aws_profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-      tags: "{{ deploy_container.aws_ecs.tags }}"
-      export: false
-      domain_name: "{{ deploy_container.aws_ecs.domain_name }}"
-      extra_domains: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
-      route_53:
-        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
-        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
-  when:
-    - deploy_container.aws_ecs.acm.create_cert
-    - deploy_container.aws_ecs.enabled
-
-- name: Default to provided SSL certificate ARN.
-  ansible.builtin.set_fact:
-    _ssl_certificate_ARN: "{{ deploy_container.aws_ecs.ssl_certificate_ARN }}"
-  when: deploy_container.aws_ecs.enabled
-
-- name: If provided, override SSL certificate ARN with the one received from ACM.
-  ansible.builtin.set_fact:
-    _ssl_certificate_ARN: "{{ aws_acm_certificate_arn }}"
-  when:
-    - deploy_container.aws_ecs.acm.create_cert
-    - deploy_container.aws_ecs.enabled
-
-- name: Define default ALB listeners.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_listeners_http:
-      Protocol: HTTP
-      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
-      DefaultActions:
-        - Type: forward
-          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
-      Rules: "{{ deploy_container.aws_ecs.elb_listener_http_rules }}"
-    _aws_ecs_cluster_listeners_redirect:
-      Protocol: HTTP
-      Port: "{{ deploy_container.aws_ecs.elb_http_port }}"
-      DefaultActions:
-        - Type: redirect
-          RedirectConfig:
-            Protocol: HTTPS
-            Host: "#{host}"
-            Query: "#{query}"
-            Path: "/#{path}"
-            Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
-            StatusCode: HTTP_301
-    _aws_ecs_cluster_listeners_https:
-      Protocol: HTTPS
-      Port: "{{ deploy_container.aws_ecs.elb_https_port }}"
-      SslPolicy: "{{ deploy_container.aws_ecs.elb_ssl_policy }}"
-      Certificates:
-        - CertificateArn: "{{ _ssl_certificate_ARN }}"
-      DefaultActions:
-        - Type: forward
-          TargetGroupName: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}"
-      Rules: "{{ deploy_container.aws_ecs.elb_listener_https_rules }}"
-  when: deploy_container.aws_ecs.enabled
-
-- name: Add HTTP listeners.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_http ] }}"
-  when:
-    - _ssl_certificate_ARN | length < 1
-    - deploy_container.aws_ecs.enabled
-
-- name: Add HTTPS Listener.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_listeners: "{{ [ _aws_ecs_cluster_listeners_redirect, _aws_ecs_cluster_listeners_https ] }}"
-  when:
-    - _ssl_certificate_ARN | length > 1
-    - deploy_container.aws_ecs.enabled
-
-- name: Add custom Listeners.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_listeners: "{{ _aws_ecs_cluster_listeners + deploy_container.aws_ecs.elb_listeners }}"
-  when:
-    - deploy_container.aws_ecs.elb_listeners is defined
-    - deploy_container.aws_ecs.elb_listeners | length
-    - deploy_container.aws_ecs.enabled
-
-- name: Create an ALB.
-  amazon.aws.elb_application_lb:
-    region: "{{ deploy_container.aws_ecs.region }}"
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    name: "{{ deploy_container.aws_ecs.target_group_name | truncate(32, true, '', 0) }}" # 32 char limit
-    state: present
-    tags: "{{ deploy_container.aws_ecs.tags }}"
-    subnets: "{{ _aws_ecs_cluster_public_subnets_ids }}"
-    security_groups: "{{ deploy_container.aws_ecs.elb_security_groups }}"
-    listeners: "{{ _aws_ecs_cluster_listeners }}"
-    idle_timeout: "{{ deploy_container.aws_ecs.elb_idle_timeout }}"
-    ip_address_type: "{{ deploy_container.aws_ecs.elb_ip_address_type }}"
-  register: _aws_ecs_cluster_alb
-  delegate_to: localhost
-  when: deploy_container.aws_ecs.enabled
-
-- name: Set task definition name.
-  ansible.builtin.set_fact:
-    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}"
-  when: deploy_container.aws_ecs.enabled
-
-- name: Set task definition revision if applicable.
-  ansible.builtin.set_fact:
-    _aws_ecs_service_task_definition: "{{ deploy_container.aws_ecs.family_name }}:{{ deploy_container.aws_ecs.task_definition_revision }}"
-  when:
-    - deploy_container.aws_ecs.task_definition_revision | length > 0
-    - deploy_container.aws_ecs.enabled
-
-- name: Create ECS service.
-  community.aws.ecs_service:
-    region: "{{ deploy_container.aws_ecs.region }}"
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    state: present
-    name: "{{ deploy_container.aws_ecs.family_name }}"
-    cluster: "{{ deploy_container.aws_ecs.cluster_name }}"
-    task_definition: "{{ _aws_ecs_service_task_definition }}"
-    desired_count: "{{ deploy_container.aws_ecs.task_count }}"
-    launch_type: "{{ deploy_container.aws_ecs.launch_type }}"
-    platform_version: LATEST
-    load_balancers: # see https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_LoadBalancer.html
-      - containerName: "{{ deploy_container.container_name }}"
-        containerPort: "{{ deploy_container.aws_ecs.target_group_port }}"
-        targetGroupArn: "{{ _aws_ecs_target_group.target_group_arn }}"
-    network_configuration:
-      subnets: "{{ _aws_ecs_cluster_private_subnets_ids }}" # internal private subnet
-      security_groups: "{{ deploy_container.aws_ecs.security_groups }}"
-      assign_public_ip: "{{ deploy_container.aws_ecs.service_public_container_ip }}"
-    tags: "{{ deploy_container.aws_ecs.tags }}"
-    enable_execute_command: "{{ deploy_container.aws_ecs.service_enable_ssm }}"
-    force_new_deployment: "{{ deploy_container.aws_ecs.service_force_refresh }}"
-    wait: true
-  delegate_to: localhost
-  when: deploy_container.aws_ecs.enabled
-
-- name: Create target tracking scaling policy for ECS service.
-  community.aws.application_autoscaling_policy:
-    region: "{{ deploy_container.aws_ecs.region }}"
-    profile: "{{ deploy_container.aws_ecs.aws_profile }}"
-    state: present
-    policy_name: "{{ deploy_container.aws_ecs.family_name }}"
-    service_namespace: ecs
-    resource_id: "service/{{ deploy_container.aws_ecs.cluster_name }}/{{ deploy_container.aws_ecs.family_name }}"
-    scalable_dimension: ecs:service:DesiredCount
-    minimum_tasks: "{{ deploy_container.aws_ecs.task_minimum_count }}"
-    maximum_tasks: "{{ deploy_container.aws_ecs.task_maximum_count }}"
-    policy_type: TargetTrackingScaling
-    target_tracking_scaling_policy_configuration:
-      PredefinedMetricSpecification:
-        PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}"
-      ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}"
-      ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}"
-      DisableScaleIn: false
-      TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}"
-  delegate_to: localhost
-  when: deploy_container.aws_ecs.enabled
-
-- name: Initialise the domains loop var with main domain entry DNS settings.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_dns_all_domains:
-      - domain: "{{ deploy_container.aws_ecs.domain_name }}"
-        zone: "{{ deploy_container.aws_ecs.route_53.zone }}"
-        aws_profile: "{{ deploy_container.aws_ecs.route_53.aws_profile }}"
-  when: deploy_container.aws_ecs.enabled
-
-- name: Add extra_domains so we can loop through DNS records.
-  ansible.builtin.set_fact:
-    _aws_ecs_cluster_dns_all_domains: "{{ _aws_ecs_cluster_dns_all_domains + [{'domain': item.domain, 'zone': item.zone, 'aws_profile': item.aws_profile}] }}"
-  loop: "{{ deploy_container.aws_ecs.acm.extra_domains }}"
-  when:
-    - deploy_container.aws_ecs.acm.extra_domains | length > 0
-    - deploy_container.aws_ecs.enabled
-
-- name: Add DNS records in Route 53.
-  amazon.aws.route53:
-    state: present
-    profile: "{{ item.aws_profile }}"
-    zone: "{{ item.zone }}"
-    record: "{{ item.domain }}"
-    type: CNAME
-    value: "{{ _aws_ecs_cluster_alb.dns_name }}"
-    overwrite: true
-  loop: "{{ _aws_ecs_cluster_dns_all_domains }}"
-  when:
-    - deploy_container.aws_ecs.route_53.zone | length > 0
-    - deploy_container.aws_ecs.enabled
\ No newline at end of file
+- name: Execute a container deployment.
+  ansible.builtin.include_tasks:
+    file: "action-{{ deploy_container.action }}.yml"
+  when: deploy_operation == 'deploy'
+
+- name: Cleanup dangling containers.
+  ansible.builtin.include_tasks:
+    file: action-cleanup.yml
+  when: deploy_operation == 'cleanup'

From 31a95fd7c14499e18263be749f9631b16e05c77c Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 11 Jan 2024 11:22:45 +0100
Subject: [PATCH 103/139] Bug fixes pr 1.x (#400)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.
---
 scripts/_common.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/_common.sh b/scripts/_common.sh
index 04e14c18..a34ff141 100755
--- a/scripts/_common.sh
+++ b/scripts/_common.sh
@@ -36,6 +36,13 @@ BUILD_TRACK_DIR="$OWN_DIR/track"
 if [ ! -d "$BUILD_TRACK_DIR" ]; then
   mkdir "$BUILD_TRACK_DIR"
 fi
+# Load the contents of profile.d in case we added items to $PATH there.
+if [ -n "$(ls -A /etc/profile.d)" ]; then
+  for f in /etc/profile.d/*; do
+  # shellcheck source=/dev/null
+    . "$f"
+  done
+fi
 ANSIBLE_LOCATION=$(command -v ansible)
 # Load the contents of profile.d in case we added items to $PATH there.
 for f in /etc/profile.d/*; do

From b7119533f53b08ad677e5c7c0aad8de4d65fe5fa Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 15 Jan 2024 13:35:05 +0100
Subject: [PATCH 104/139] Bug fixes pr 1.x (#402)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.

* Adding support for specifying path to Ansible and path to Python interpreter.
---
 scripts/_common.sh | 21 ++++++++++++++++++++-
 scripts/build.sh   |  2 ++
 scripts/cleanup.sh |  2 ++
 scripts/deploy.sh  |  2 ++
 scripts/revert.sh  |  2 ++
 5 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/scripts/_common.sh b/scripts/_common.sh
index a34ff141..07aa7368 100755
--- a/scripts/_common.sh
+++ b/scripts/_common.sh
@@ -14,6 +14,8 @@ PREVIOUS_BUILD_NUMBER=""
 CURRENT_BUILD_NUMBER=""
 ANSIBLE_EXTRA_VARS=""
 ANSIBLE_DEFAULT_EXTRA_VARS=""
+ANSIBLE_PATH=""
+PYTHON_INTERPRETER=""
 BUILD_WORKSPACE=""
 BUILD_TRACK_FILE=""
 BUILD_ID=""
@@ -103,6 +105,14 @@ parse_options(){
           shift
           BOTO_PROFILE="$1"
         ;;
+      "--ansible-path")
+          shift
+          ANSIBLE_PATH="$1"
+        ;;
+      "--python-interpreter")
+          shift
+          PYTHON_INTERPRETER="$1"
+        ;;
       "--build-id")
           shift
           BUILD_ID="$1"
@@ -128,7 +138,11 @@ get_build_workspace(){
 
 # Common extra-vars to pass to Ansible.
 get_ansible_defaults_vars(){
-  ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}"
+  if [ -n "$PYTHON_INTERPRETER" ]; then
+    ANSIBLE_DEFAULT_EXTRA_VARS="{ansible_python_interpreter: $PYTHON_INTERPRETER, _ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}"
+  else
+    ANSIBLE_DEFAULT_EXTRA_VARS="{_ce_deploy_base_dir: $OWN_DIR, _ce_deploy_build_dir: $BUILD_WORKSPACE, _ce_deploy_build_tmp_dir: $BUILD_TMP_DIR, _ce_deploy_data_dir: $ANSIBLE_DATA_DIR, _ce_deploy_ansible_location: $ANSIBLE_LOCATION, build_number: $CURRENT_BUILD_NUMBER, previous_known_build_number: $PREVIOUS_BUILD_NUMBER}"
+  fi
 }
 
 # Fetch previous build number from track file.
@@ -169,6 +183,11 @@ cleanup_build_tmp_dir(){
 # Call Ansible playbook to ensure host exists.
 ansible_host_check(){
   if [ -n "$TARGET_DEPLOY_HOST" ]; then
+    if [ -z "$ANSIBLE_PATH" ]; then
+      ANSIBLE_BIN=$(command -v ansible-playbook)
+    else
+      ANSIBLE_BIN="$ANSIBLE_PATH/ansible-playbook"
+    fi
     ANSIBLE_BIN=$(command -v ansible-playbook)
     ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml"
     if [ "$VERBOSE" = "yes" ]; then
diff --git a/scripts/build.sh b/scripts/build.sh
index 93f36887..a29ec43b 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -21,6 +21,8 @@ usage(){
   echo 'Available options:'
   echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.'
   echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.'
+  echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.'
+  echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.'
   echo '--previous-stable-build-number: an incremental build number that '
   echo '--dry-run: Do not perform any action but run the playbooks in --check mode.'
   echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'
diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh
index 9f0e50cf..abe24675 100755
--- a/scripts/cleanup.sh
+++ b/scripts/cleanup.sh
@@ -20,6 +20,8 @@ usage(){
   echo ''
   echo 'Available options:'
   echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.'
+  echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.'
+  echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.'
   echo '--previous-stable-build-number: an incremental build number that '
   echo '--dry-run: Do not perform any action but run the playbooks in --check mode.'
   echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index 341af1ce..66ab7fe3 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -20,6 +20,8 @@ usage(){
   echo ''
   echo 'Available options:'
   echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.'
+  echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.'
+  echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.'
   echo '--previous-stable-build-number: an incremental build number that '
   echo '--dry-run: Do not perform any action but run the playbooks in --check mode.'
   echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'
diff --git a/scripts/revert.sh b/scripts/revert.sh
index a10655a3..7b36b42b 100755
--- a/scripts/revert.sh
+++ b/scripts/revert.sh
@@ -20,6 +20,8 @@ usage(){
   echo ''
   echo 'Available options:'
   echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.'
+  echo '--ansible-path: Pass the path to the directory containing the Ansible binaries if you are not using the version of Ansible in PATH.'
+  echo '--python-interpreter: When using Python virtual environments Ansible may not correctly determine the Python interpreter, use this to set it manually.'
   echo '--previous-stable-build-number: an incremental build number that '
   echo '--dry-run: Do not perform any action but run the playbooks in --check mode.'
   echo '--verbose: Detailled informations. This can potentially leak sensitive information in the output'

From 6a89c547c84575b2f7a4311a3bdbd214354b994e Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 19 Jan 2024 13:14:39 +0200
Subject: [PATCH 105/139] r67637-terminate-sessions-keeping-loop-device-busy
 (#405)

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy

* r67637-terminate-sessions-keeping-loop-device-busy
---
 roles/deploy_code/tasks/cleanup.yml | 40 +++++++++++++++++++----------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 776b0557..02065061 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -1,18 +1,4 @@
 ---
-- name: Build warning.
-  ansible.builtin.debug:
-    msg: "### WARNING - if you are using a server terminal exit NOW! You have 20 seconds to comply..."
-  run_once: true
-  when:
-    - deploy_code.mount_type == "squashfs"
-
-- name: Give people 20 seconds to exit terminal sessions.
-  ansible.builtin.pause:
-    seconds: 20
-  run_once: true
-  when:
-    - deploy_code.mount_type == "squashfs"
-
 - name: Ensure codebase is writable.
   ansible.builtin.shell:
     cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi"
@@ -155,6 +141,32 @@
       failed_when: false
       register: _deploy_code_mount_check
 
+    - name: Get the current pts session.
+      ansible.builtin.shell:
+        cmd: "tty | sed 's#/dev/##'"
+      register: deploy_pts
+
+    - name: "Check for active sessions in {{ deploy_base_path }}."
+      ansible.builtin.shell:
+        cmd: "ps -eo pid,tty | awk '{print $1}' | xargs -n 1 pwdx 2>&1 | grep -v 'No such process' | grep {{ deploy_base_path }} | cut -d: -f1 | xargs -n 1 ps -o tty= -p | sort | uniq"
+      register: sessions_in_deploy_path
+      become: true
+
+    - name: Kill sessions except the current one.
+      ansible.builtin.command:
+        cmd: "pkill -9 -t {{ item }}"
+      loop: "{{ sessions_in_deploy_path.stdout_lines }}"
+      when: "item != deploy_pts.stdout"
+      failed_when: false
+      register: kill_sessions_result
+      become: true
+
+    - name: Display session info.
+      ansible.builtin.debug:
+        msg: >
+          Deploy session: {{ deploy_pts.stdout }}.
+          Sessions terminated: {{ kill_sessions_result.results | selectattr('rc', 'equalto', 0) | map(attribute='item') | list | join(', ') | default('None') }}.
+
     - name: Reload any services that might be keeping the loop device busy.
       ansible.builtin.service:
         name: "{{ www_service }}"

From 4a777a489a545af71ead9911da68588732b6f193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matej=20=C5=A0tajduhar?=
 <30931414+matej5@users.noreply.github.com>
Date: Fri, 19 Jan 2024 15:45:55 +0100
Subject: [PATCH 106/139] Avoiding-cache-clear-warning-for-site-variable (#415)

Co-authored-by: Matej Stajduhar <matej.stajduhar@codeenigma.com>
---
 roles/cache_clear/cache_clear-drupal7/tasks/main.yml | 6 +++---
 roles/cache_clear/cache_clear-drupal8/tasks/main.yml | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
index 6f72b3a5..b68e1979 100644
--- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml
@@ -1,11 +1,11 @@
 ---
 - name: Clear Drupal 7 cache.
   command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all"
-    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site_drupal.folder }} -y cc all"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   with_items: "{{ drupal.sites }}"
   loop_control:
-    loop_var: site
+    loop_var: site_drupal
   run_once: true
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index 5289f254..92db6584 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,13 +1,13 @@
 ---
 - name: Clear Drupal cache.
   ansible.builtin.command:
-    cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr"
-    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+    cmd: "{{ drush_bin }} -l {{ site_drupal.folder }} -y cr"
+    chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   with_items: "{{ drupal.sites }}"
   loop_control:
-    loop_var: site
+    loop_var: site_drupal
   run_once: true
   register: _drush_output
 

From 27a7160b5943315dd0005dac3ff90916f5f8be34 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Mon, 22 Jan 2024 12:55:14 +0200
Subject: [PATCH 107/139] additional check and debug task (#417)

* additional check and debug task

* additional check and debug task

* additional check and debug task

* additional_checks_and_moving_debug_task_to_simplify_diagnostics
---
 roles/deploy_code/tasks/cleanup.yml | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index 02065061..f3fc2c73 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -152,20 +152,27 @@
       register: sessions_in_deploy_path
       become: true
 
+    - name: Display active sessions.
+      ansible.builtin.debug:
+        msg: >
+          Deploy session: {{ deploy_pts.stdout | default('Unknown') }}.
+          Active sessions in {{ deploy_base_path }}: {{ sessions_in_deploy_path.stdout_lines | default([]) | join(', ') | default('None') }}.
+
     - name: Kill sessions except the current one.
       ansible.builtin.command:
         cmd: "pkill -9 -t {{ item }}"
       loop: "{{ sessions_in_deploy_path.stdout_lines }}"
-      when: "item != deploy_pts.stdout"
+      when:
+        - "item != deploy_pts.stdout"
+        - "item is match('^pts/\\d+$')"
       failed_when: false
       register: kill_sessions_result
       become: true
 
-    - name: Display session info.
+    - name: Display killed sessions.
       ansible.builtin.debug:
         msg: >
-          Deploy session: {{ deploy_pts.stdout }}.
-          Sessions terminated: {{ kill_sessions_result.results | selectattr('rc', 'equalto', 0) | map(attribute='item') | list | join(', ') | default('None') }}.
+          Sessions terminated: {{ kill_sessions_result.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | map(attribute='item') | list | join(', ') | default('None') }}.
 
     - name: Reload any services that might be keeping the loop device busy.
       ansible.builtin.service:

From 840863442bf2dd11d78485b1d5e82c19734e52d2 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Wed, 24 Jan 2024 18:04:54 +0200
Subject: [PATCH 108/139] r65896-cache-clear-path (#422)

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* grouping squashfs tasks

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir

* r65896-cache-clear-dir
---
 roles/_init/tasks/drupal8.yml                 |   2 +-
 roles/_init/tasks/main.yml                    |  74 +++++-----
 .../cache_clear-drupal8/tasks/main.yml        |  13 ++
 roles/deploy_code/tasks/cleanup.yml           | 126 +++++++-----------
 4 files changed, 101 insertions(+), 114 deletions(-)

diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml
index 3792c489..478b4426 100644
--- a/roles/_init/tasks/drupal8.yml
+++ b/roles/_init/tasks/drupal8.yml
@@ -5,4 +5,4 @@
 
 - name: Define path to drush for currently live build.
   set_fact:
-    drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}"
\ No newline at end of file
+    drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}"
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index c054e8fb..225e9b60 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -3,27 +3,35 @@
 - name: Define deploy user.
   ansible.builtin.set_fact:
     deploy_user: "{{ deploy_user | default('deploy') }}"
+
 - name: Define deploy base path.
   ansible.builtin.set_fact:
     deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}"
+
 - name: Define mounted directory for assets.
   ansible.builtin.set_fact:
     deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}"
+
 - name: Define webroot.
   ansible.builtin.set_fact:
     webroot: "{{ webroot | default('web') }}"
+
 - name: Define build deploy path prefix.
   ansible.builtin.set_fact:
     deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_"
+
 - name: Define build deploy path.
   ansible.builtin.set_fact:
     deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}"
+
 - name: Define live_symlink dest.
   ansible.builtin.set_fact:
     live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
+
 - name: Define opcache cachetool path.
   ansible.builtin.set_fact:
     cachetool_bin: "{{ deploy_base_path }}/cachetool.phar"
+
 - name: Set opcache cachetool path from variable if provided.
   ansible.builtin.set_fact:
     cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}"
@@ -31,38 +39,37 @@
     - cache_clear_opcache.cachetool_bin is defined
     - cache_clear_opcache.cachetool_bin | length > 0
 
-# Manipulate variables for SquashFS builds.
-- name: Define image builds base path.
-  ansible.builtin.set_fact:
-    build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}"
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-- name: Define image builds build path prefix.
-  ansible.builtin.set_fact:
-    build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_"
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-- name: Define live_symlink dest for image builds.
-  ansible.builtin.set_fact:
-    live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-- name: Overwrite deploy and live_symlink paths if SquashFS deploy.
-  ansible.builtin.set_fact:
-    deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-- name: Overwrite cachetool path if SquashFS deploy and path not provided.
-  ansible.builtin.set_fact:
-    cachetool_bin: "{{ build_base_path }}/cachetool.phar"
+- name: Manipulate variables for SquashFS builds.
+  block:
+    - name: Define image builds base path.
+      ansible.builtin.set_fact:
+        build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}"
+
+    - name: Define image builds build path prefix.
+      ansible.builtin.set_fact:
+        build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_"
+
+    - name: Define live_symlink dest for image builds.
+      ansible.builtin.set_fact:
+        live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
+
+    - name: Overwrite deploy and live_symlink paths if SquashFS deploy.
+      ansible.builtin.set_fact:
+        deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
+
+    - name: Overwrite cachetool path if SquashFS deploy and path not provided.
+      ansible.builtin.set_fact:
+        cachetool_bin: "{{ build_base_path }}/cachetool.phar"
+      when:
+        - cache_clear_opcache.cachetool_bin is not defined
+
+    - name: Ensure build target directory exists.
+      ansible.builtin.file:
+        path: "{{ build_base_path }}"
+        state: directory
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
-    - cache_clear_opcache.cachetool_bin is not defined
 
 # Gather last known good build directly from symlink.
 # This can happen:
@@ -99,15 +106,6 @@
     path: "{{ deploy_base_path }}"
     state: directory
 
-# Make sure the build target exists.
-- name: Ensure build target directory exists.
-  ansible.builtin.file:
-    path: "{{ build_base_path }}"
-    state: directory
-  when:
-    - deploy_code.mount_type is defined
-    - deploy_code.mount_type == "squashfs"
-
 # Check for project specific init tasks.
 - name: Check that {{ project_type }}.yml exists.
   ansible.builtin.stat:
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index 92db6584..1fc9d2f2 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -1,4 +1,17 @@
 ---
+- name: Override paths for squashfs build cleanup step.
+  block:
+    - name: Override path to drush binary.
+      set_fact:
+        drush_bin: "{{ drush_live_bin }}"
+    - name: Override deploy_path.
+      set_fact:
+        deploy_path: "{{ live_symlink_dest }}"
+  when:
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
+    - deploy_operation == 'cleanup'
+
 - name: Clear Drupal cache.
   ansible.builtin.command:
     cmd: "{{ drush_bin }} -l {{ site_drupal.folder }} -y cr"
diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml
index f3fc2c73..449c1103 100644
--- a/roles/deploy_code/tasks/cleanup.yml
+++ b/roles/deploy_code/tasks/cleanup.yml
@@ -17,15 +17,6 @@
     - deploy_code.perms_fix_path | length > 1
     - deploy_code.mount_type != "squashfs"
 
-- name: Ensure permissions are set on builds directory.
-  ansible.builtin.shell:
-    cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
-  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
-  when:
-    - deploy_code.perms_fix_path is defined
-    - deploy_code.perms_fix_path | length > 1
-    - deploy_code.mount_type == "squashfs"
-
 - name: Delete codebases from deploy directory.
   ansible.builtin.file:
     name: "{{ deploy_path_prefix }}{{ item }}"
@@ -34,14 +25,6 @@
   when:
     - deploy_code.mount_type != "squashfs"
 
-- name: Delete codebases from builds directory.
-  ansible.builtin.file:
-    name: "{{ build_path_prefix }}{{ item }}"
-    state: absent
-  with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
-  when:
-    - deploy_code.mount_type == "squashfs"
-
 - name: Create a tarball of the deployed codebases.
   ansible.builtin.command:
     cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}"
@@ -51,15 +34,6 @@
     - deploy_code.mount_type == "tarball"
   run_once: true
 
-- name: Create a SquashFS image of the deployed codebases.
-  ansible.builtin.command:
-    cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh"
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-  run_once: true
-
 - name: Create destination folder.
   ansible.builtin.file:
     path: "{{ deploy_code.mount_sync }}"
@@ -79,60 +53,57 @@
     - deploy_code.mount_type == "tarball"
   run_once: true
 
-- name: Move SquashFS image to final destination.
-  ansible.builtin.command:
-    cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh"
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-  run_once: true
+# Beginning of the squashFS block.
+- name: Execute tasks for squashFS mount type.
+  block:
+    - name: Ensure permissions are set on builds directory.
+      ansible.builtin.shell:
+        cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi"
+      with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
+      when:
+        - deploy_code.perms_fix_path is defined
+        - deploy_code.perms_fix_path | length > 1
 
-- name: Check if we have a SquashFS image already there.
-  ansible.builtin.stat:
-    path: "{{ build_base_path }}/deploy.sqsh"
-  register: _deploy_code_mount_image
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
+    - name: Delete codebases from builds directory.
+      ansible.builtin.file:
+        name: "{{ build_path_prefix }}{{ item }}"
+        state: absent
+      with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - deploy_code.keep, 0] | max }}
 
-- name: Copy previous SquashFS image to shared mount point in case of rollback.
-  ansible.builtin.copy:
-    remote_src: true
-    force: true
-    src: "{{ build_base_path }}/deploy.sqsh"
-    dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh"
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-    - _deploy_code_mount_image.stat.islnk is defined
-  run_once: true
+    - name: Create a SquashFS image of the deployed codebases.
+      ansible.builtin.command:
+        cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh"
+      run_once: true
 
-- name: Ensure mounted SquashFS image is deleted.
-  ansible.builtin.file:
-    path: "{{ build_base_path }}/deploy.sqsh"
-    state: absent
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
+    - name: Move SquashFS image to final destination.
+      ansible.builtin.command:
+        cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh"
+      run_once: true
+
+    - name: Check if we have a SquashFS image already there.
+      ansible.builtin.stat:
+        path: "{{ build_base_path }}/deploy.sqsh"
+      register: _deploy_code_mount_image
+
+    - name: Copy previous SquashFS image to shared mount point in case of rollback.
+      ansible.builtin.copy:
+        remote_src: true
+        force: true
+        src: "{{ build_base_path }}/deploy.sqsh"
+        dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh"
+      when:
+        - _deploy_code_mount_image.stat.islnk is defined
+      run_once: true
 
-- name: Copy SquashFS image to local server.
-  ansible.builtin.command:
-    cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh"
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
+    - name: Ensure mounted SquashFS image is deleted.
+      ansible.builtin.file:
+        path: "{{ build_base_path }}/deploy.sqsh"
+        state: absent
+
+    - name: Copy SquashFS image to local server.
+      ansible.builtin.command:
+        cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh"
 
-- name: Unmount and remount squashfs images.
-  when:
-    - deploy_code.mount_sync is defined
-    - deploy_code.mount_sync | length > 1
-    - deploy_code.mount_type == "squashfs"
-  block:
     - name: Check if we have a mount already.
       ansible.builtin.shell:
         cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}"
@@ -224,6 +195,11 @@
         - _deploy_code_mount_check.rc == 0
         - deploy_code.service_action == "stop"
         - deploy_code.services | length > 0
+  when:
+    - deploy_code.mount_sync is defined
+    - deploy_code.mount_sync | length > 1
+    - deploy_code.mount_type == "squashfs"
+# End of the squashFS block.
 
 - name: Trigger an infrastructure rebuild.
   ansible.builtin.include_role:

From 53d1f9c87c348c7dfc13d1307fa546732a6775e5 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 30 Jan 2024 13:28:46 +0200
Subject: [PATCH 109/139] r65896-cache-container-table-truncate (#434)

* r65896-cache-container-table-truncate

* r65896-cache-container-table-truncate

* r65896-cache-container-table-truncate

* roles/_init/defaults/main.yml
---
 roles/_init/defaults/main.yml                 |  2 ++
 .../cache_clear-drupal8/tasks/main.yml        | 21 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 5fcee089..2e4eb0a8 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -7,6 +7,8 @@ deploy_user: "deploy"
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   drush_verbose_output: false
+  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
+  truncate_cache_table: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index 1fc9d2f2..9a02f48d 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -4,9 +4,30 @@
     - name: Override path to drush binary.
       set_fact:
         drush_bin: "{{ drush_live_bin }}"
+
     - name: Override deploy_path.
       set_fact:
         deploy_path: "{{ live_symlink_dest }}"
+
+    - name: Truncate table cache_container.
+      ansible.builtin.command:
+        cmd: "{{ drush_bin }} sql-query 'TRUNCATE TABLE cache_container'"
+        chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}"
+      become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+      become_user: "{{ www_user }}"
+      with_items: "{{ drupal.sites }}"
+      loop_control:
+        loop_var: site_drupal
+      run_once: true
+      register: _drush_output
+      when: drupal.truncate_cache_table
+
+    - name: Show drush output.
+      ansible.builtin.debug:
+        msg: "{{ _drush_output }}"
+      when:
+        - drupal.drush_verbose_output
+        - drupal.truncate_cache_table
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"

From 575b0cc0fc8cdec18740f6287cb93538c17a6010 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Wed, 31 Jan 2024 16:42:04 +0200
Subject: [PATCH 110/139] 
 r65896-cache-container-table-truncate-change-tasks-order (#439)

---
 .../cache_clear-drupal8/tasks/main.yml        | 45 ++++++++++---------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index 9a02f48d..a0f960da 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -8,26 +8,6 @@
     - name: Override deploy_path.
       set_fact:
         deploy_path: "{{ live_symlink_dest }}"
-
-    - name: Truncate table cache_container.
-      ansible.builtin.command:
-        cmd: "{{ drush_bin }} sql-query 'TRUNCATE TABLE cache_container'"
-        chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}"
-      become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
-      become_user: "{{ www_user }}"
-      with_items: "{{ drupal.sites }}"
-      loop_control:
-        loop_var: site_drupal
-      run_once: true
-      register: _drush_output
-      when: drupal.truncate_cache_table
-
-    - name: Show drush output.
-      ansible.builtin.debug:
-        msg: "{{ _drush_output }}"
-      when:
-        - drupal.drush_verbose_output
-        - drupal.truncate_cache_table
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
@@ -49,3 +29,28 @@
   ansible.builtin.debug:
     msg: "{{ _drush_output }}"
   when: drupal.drush_verbose_output
+
+- name: Truncate SQL table 'cache_container'.
+  block:
+    - name: Truncate table 'cache_container'.
+      ansible.builtin.command:
+        cmd: "{{ drush_bin }} sql-query 'TRUNCATE TABLE cache_container'"
+        chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site_drupal.folder }}"
+      become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+      become_user: "{{ www_user }}"
+      with_items: "{{ drupal.sites }}"
+      loop_control:
+        loop_var: site_drupal
+      run_once: true
+      register: _drush_output
+
+    - name: Show drush output.
+      ansible.builtin.debug:
+        msg: "{{ _drush_output }}"
+      when:
+        - drupal.drush_verbose_output
+  when:
+    - drupal.truncate_cache_table
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
+    - deploy_operation == 'cleanup'

From 4c9ab4e3027a765b41fa98718c9ebf146a51cb9c Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 31 Jan 2024 19:10:07 +0100
Subject: [PATCH 111/139] Bug fixes pr 1.x (#441)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.

* Adding support for specifying path to Ansible and path to Python interpreter.

* Moving installation of cachetool to application includes, where they are PHP apps.
---
 roles/_init/tasks/drupal7.yml | 5 +++++
 roles/_init/tasks/drupal8.yml | 5 +++++
 roles/_init/tasks/main.yml    | 5 -----
 roles/_init/tasks/matomo.yml  | 6 ++++--
 roles/_init/tasks/mautic.yml  | 6 ++++--
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml
index b1781c1b..3053c5c4 100644
--- a/roles/_init/tasks/drupal7.yml
+++ b/roles/_init/tasks/drupal7.yml
@@ -10,3 +10,8 @@
 - name: Ensure we have a drush binary.
   import_role:
     name: cli/drush
+
+- name: Ensure we have a cachetool binary.
+  ansible.builtin.import_role:
+    name: cli/cachetool
+  when: install_php_cachetool
diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml
index 478b4426..7c64719e 100644
--- a/roles/_init/tasks/drupal8.yml
+++ b/roles/_init/tasks/drupal8.yml
@@ -6,3 +6,8 @@
 - name: Define path to drush for currently live build.
   set_fact:
     drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}"
+
+- name: Ensure we have a cachetool binary.
+  ansible.builtin.import_role:
+    name: cli/cachetool
+  when: install_php_cachetool
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 225e9b60..f57be10e 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -118,8 +118,3 @@
   ansible.builtin.include_tasks: "{{ project_type }}.yml"
   when:
     - _project_type_task_result.stat.exists
-
-- name: Ensure we have a cachetool binary.
-  ansible.builtin.import_role:
-    name: cli/cachetool
-  when: install_php_cachetool
diff --git a/roles/_init/tasks/matomo.yml b/roles/_init/tasks/matomo.yml
index 6f34c92f..32f693a1 100644
--- a/roles/_init/tasks/matomo.yml
+++ b/roles/_init/tasks/matomo.yml
@@ -1,3 +1,5 @@
 ---
-
-# Nothing to do here.
\ No newline at end of file
+- name: Ensure we have a cachetool binary.
+  ansible.builtin.import_role:
+    name: cli/cachetool
+  when: install_php_cachetool
diff --git a/roles/_init/tasks/mautic.yml b/roles/_init/tasks/mautic.yml
index 6f34c92f..32f693a1 100644
--- a/roles/_init/tasks/mautic.yml
+++ b/roles/_init/tasks/mautic.yml
@@ -1,3 +1,5 @@
 ---
-
-# Nothing to do here.
\ No newline at end of file
+- name: Ensure we have a cachetool binary.
+  ansible.builtin.import_role:
+    name: cli/cachetool
+  when: install_php_cachetool

From 3df00305619a2a059194be2b64dd0b0fdd34a9b6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 8 Feb 2024 14:12:21 +0100
Subject: [PATCH 112/139] Adding a pause if a replica is declared, so the
 replica can catch up with the writer. (#443)

---
 docs/roles/_init.md                           |  2 ++
 .../database_backup/database_backup-mysql.md  | 19 ++++++++++++++++++-
 roles/_init/README.md                         |  2 ++
 .../database_backup-mysql/README.md           | 19 ++++++++++++++++++-
 .../database_backup-mysql/defaults/main.yml   |  4 +++-
 .../tasks/deploy-none.yml                     |  7 +++++++
 6 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index d561aac2..98179138 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -16,6 +16,8 @@ deploy_user: "deploy"
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   drush_verbose_output: false
+  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
+  truncate_cache_table: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index 1200f97f..b2c79799 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -1,5 +1,20 @@
 # MySQL backups
 Generate MySQL backups for each build.
+
+## Replicas
+If you are using a read only replica in your application and you need to add it to `databases` in order to have access to the credentials for your app settings, be sure to set the database up in a similar way to this:
+
+```yaml
+mysql_backup:
+  databases:
+    - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}"
+      user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}"
+      credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
+      handling: none # prevents the replica from being backed up
+      is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause
+      pause_seconds: 30 # duration of the pause in seconds
+```
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
@@ -27,7 +42,9 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
-      #handling: static # optional override to the main handling method on a per database basis
+      #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
+      #is_replica: true # tell ce-deploy this database is a replica
+      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
 
 ```
 
diff --git a/roles/_init/README.md b/roles/_init/README.md
index d561aac2..98179138 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -16,6 +16,8 @@ deploy_user: "deploy"
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
 drupal:
   drush_verbose_output: false
+  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
+  truncate_cache_table: false
   sites:
     - folder: "default"
       public_files: "sites/default/files"
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index 1200f97f..b2c79799 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -1,5 +1,20 @@
 # MySQL backups
 Generate MySQL backups for each build.
+
+## Replicas
+If you are using a read only replica in your application and you need to add it to `databases` in order to have access to the credentials for your app settings, be sure to set the database up in a similar way to this:
+
+```yaml
+mysql_backup:
+  databases:
+    - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}"
+      user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}"
+      credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
+      handling: none # prevents the replica from being backed up
+      is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause
+      pause_seconds: 30 # duration of the pause in seconds
+```
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
@@ -27,7 +42,9 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
-      #handling: static # optional override to the main handling method on a per database basis
+      #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
+      #is_replica: true # tell ce-deploy this database is a replica
+      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
 
 ```
 
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 1091f8d7..23fa413a 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -22,4 +22,6 @@ mysql_backup:
     - database: "{{ (project_name + '_' + build_type) | regex_replace('-', '_') }}" # avoid hyphens in MySQL database names
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
-      #handling: static # optional override to the main handling method on a per database basis
+      #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
+      #is_replica: true # tell ce-deploy this database is a replica
+      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index 4c4b1624..aa8c7317 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -4,6 +4,13 @@
 - ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
+- name: Pause to allow replication to catch up.
+  ansible.builtin.pause:
+    seconds: "{{ database.pause_seconds }}"
+  when:
+    - database.is_replica is defined
+    - database.is_replica
+
 - name: Check if the database exists already.
   ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}"
   register: _build_db_status

From ddab8aab50be85f3840e0fc907fb21a652d2d0d8 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 22 Feb 2024 14:08:21 +0100
Subject: [PATCH 113/139] Db replica support pr 1.x (#445)

* Adding a pause if a replica is declared, so the replica can catch up with the writer.

* Making read replica support more robust.
---
 .../database_backup-mysql/tasks/cleanup-rolling.yml   |  2 ++
 .../database_backup-mysql/tasks/deploy-dump.yml       |  4 +++-
 .../database_backup-mysql/tasks/deploy-none.yml       | 11 +++--------
 .../database_backup-mysql/tasks/deploy-rolling.yml    |  2 ++
 .../database_backup-mysql/tasks/revert-dump.yml       |  4 +++-
 5 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
index 374e65ca..6d132cd5 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
@@ -8,3 +8,5 @@
     login_password: "{{ _mysql_password }}"
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
+  when:
+    - database.is_replica is not defined
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 49bda27e..32108e0a 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -20,7 +20,9 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when: previous_build_number == 0 or _build_db_status.rc == 1
+  when:
+    - previous_build_number == 0 or _build_db_status.rc == 1
+    - database.is_replica is not defined
   run_once: true
 
 - name: Ensure the dump directory exists.
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index aa8c7317..fadc79bb 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -4,13 +4,6 @@
 - ansible.builtin.set_fact:
     _mysql_previous_build_database_name: "{{ database.database }}"
 
-- name: Pause to allow replication to catch up.
-  ansible.builtin.pause:
-    seconds: "{{ database.pause_seconds }}"
-  when:
-    - database.is_replica is defined
-    - database.is_replica
-
 - name: Check if the database exists already.
   ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_build_database_name }}"
   register: _build_db_status
@@ -27,5 +20,7 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when: previous_build_number == 0 or _build_db_status.rc == 1
+  when:
+    - previous_build_number == 0 or _build_db_status.rc == 1
+    - database.is_replica is not defined
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 30fbaa62..38e82fa4 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -20,6 +20,8 @@
     config_file: "{{ database.credentials_file }}"
     config_overrides_defaults: true
   run_once: true
+  when:
+    - database.is_replica is not defined
 
 - name: Check if the previous database exists.
   ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_previous_build_database_name }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index 6040c73a..e99a8bda 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -19,7 +19,9 @@
   args:
     executable: /bin/bash
   run_once: true
-  when: previous_build_number > 0
+  when:
+    - previous_build_number > 0
+    - database.is_replica is not defined
 
 - name: Delete unpacked dump file.
   ansible.builtin.file:

From df0e68ac6008c4bd5e8aa542e35a7ab67722da8a Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 23 Feb 2024 17:04:47 +0100
Subject: [PATCH 114/139] Db replica support pr 1.x (#447)

* Adding a pause if a replica is declared, so the replica can catch up with the writer.

* Making read replica support more robust.

* Ensuring mysql backup takes no action on replicas.

* Suppressing backup jobs for replica databases.

* Minor docs update.
---
 .../database_backup/database_backup-mysql.md  |  4 +---
 .../cron_database_backup-mysql/tasks/main.yml |  4 ++--
 .../tasks/setup.yml                           | 22 ++++++++++---------
 .../database_backup-mysql/README.md           |  4 +---
 .../database_backup-mysql/defaults/main.yml   |  3 +--
 .../tasks/cleanup-rolling.yml                 |  2 --
 .../database_backup-mysql/tasks/cleanup.yml   |  5 ++++-
 .../tasks/deploy-dump.yml                     |  4 +---
 .../tasks/deploy-none.yml                     |  4 +---
 .../tasks/deploy-rolling.yml                  |  2 --
 .../database_backup-mysql/tasks/deploy.yml    | 10 +++++++--
 .../tasks/revert-dump.yml                     |  4 +---
 .../database_backup-mysql/tasks/revert.yml    |  1 +
 13 files changed, 33 insertions(+), 36 deletions(-)

diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index b2c79799..2766a4f3 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -12,7 +12,6 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       handling: none # prevents the replica from being backed up
       is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause
-      pause_seconds: 30 # duration of the pause in seconds
 ```
 
 <!--ROLEVARS-->
@@ -43,8 +42,7 @@ mysql_backup:
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
-      #is_replica: true # tell ce-deploy this database is a replica
-      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
+      #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required
 
 ```
 
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
index ffd14157..e87cad12 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml
@@ -4,7 +4,7 @@
 # add a keep mechanism for backup scripts, like for the dumps themselves.
 # Nice to have more than anything.
 - name: Setup PATH in crontab.
-  cron:
+  ansible.builtin.cron:
     name: PATH
     env: true
     job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin"
@@ -13,7 +13,7 @@
     - drupal.defer is defined
     - drupal.defer
 
-- include_tasks: setup.yml
+- ansible.builtin.include_tasks: setup.yml
   vars:
     database: database
   with_items: "{{ build_databases }}"
diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
index d09fbbac..f12301da 100644
--- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
+++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml
@@ -1,39 +1,40 @@
 # Probably not worth adding more granularity (weeks, months) for backups, should be at least daily?
-- set_fact:
+- ansible.builtin.set_fact:
     _cron_mysql_backup_random_minute: "{{ 59 | random }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _cron_mysql_backup_minute: "{{ database.original.backup.minute | default(_cron_mysql_backup_random_minute) }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _cron_mysql_backup_hour: "{{ database.original.backup.hour | default(0) }}"
-- set_fact:
+- ansible.builtin.set_fact:
     _cron_mysql_backup_keep: "{{ database.original.backup.keep | default(10) }}"
 
 - name: Ensure backup directory exists.
-  file:
+  ansible.builtin.file:
     path: "{{ cron_mysql_backup.dumps_directory }}/{{ database.host }}"
     state: directory
     mode: 0700
     recurse: true
 
 - name: Ensure cron directory exists.
-  file:
+  ansible.builtin.file:
     path: "/home/{{ deploy_user }}/cron/{{ project_name }}_{{ build_type }}"
     state: directory
     mode: 0700
     recurse: true
 
 - name: Create backup script.
-  template:
+  ansible.builtin.template:
     src: "regular-backups.sh.j2"
     dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
     mode: 0700
+  when: database.is_replica is not defined
 
 - name: Define backup cron job command.
-  set_fact:
+  ansible.builtin.set_fact:
     _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh"
 
 - name: Define backup cron job command if deferred (ASG).
-  set_fact:
+  ansible.builtin.set_fact:
     _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\""
   when:
     - drupal.defer is defined
@@ -42,9 +43,10 @@
     - drupal.defer_target | length > 0
 
 - name: Setup regular backup for MySQL.
-  cron:
+  ansible.builtin.cron:
     name: "cron_mysql_{{ database.host }}_{{ database.original.database }}"
     minute: "{{ _cron_mysql_backup_minute }}"
     hour: "{{ _cron_mysql_backup_hour }}"
     job: "{{ _backup_cron_job_command }}"
   delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}"
+  when: database.is_replica is not defined
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index b2c79799..2766a4f3 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -12,7 +12,6 @@ mysql_backup:
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       handling: none # prevents the replica from being backed up
       is_replica: true # tells ce-deploy we are working with a replica, so it will implement a pause
-      pause_seconds: 30 # duration of the pause in seconds
 ```
 
 <!--ROLEVARS-->
@@ -43,8 +42,7 @@ mysql_backup:
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
-      #is_replica: true # tell ce-deploy this database is a replica
-      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
+      #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required
 
 ```
 
diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index 23fa413a..e7351d89 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -23,5 +23,4 @@ mysql_backup:
       user: "{{ (project_name + '_' + build_type) | truncate(32, true, '', 0) }}" # 32 char limit
       credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
       #handling: none # optional override to the main handling method on a per database basis - must be 'none' for replicas
-      #is_replica: true # tell ce-deploy this database is a replica
-      #pause_seconds: 30 # how long to allow for replication to catch up, required if 'is_replica' is set to 'true'
+      #is_replica: true # tell ce-deploy this database is a replica - can only be true, remove/comment out if not required
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
index 6d132cd5..374e65ca 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
@@ -8,5 +8,3 @@
     login_password: "{{ _mysql_password }}"
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
-  when:
-    - database.is_replica is not defined
diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
index 89cbf171..23654247 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml
@@ -14,6 +14,7 @@
     _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}"
 
 - ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml"
+  when: database.is_replica is not defined
 
 - name: Delete mysql users.
   community.mysql.mysql_user:
@@ -24,5 +25,7 @@
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
-  when: mysql_backup.credentials_handling == 'rotate'
+  when:
+    - mysql_backup.credentials_handling == 'rotate'
+    - database.is_replica is not defined
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 32108e0a..49bda27e 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -20,9 +20,7 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when:
-    - previous_build_number == 0 or _build_db_status.rc == 1
-    - database.is_replica is not defined
+  when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true
 
 - name: Ensure the dump directory exists.
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index fadc79bb..4c4b1624 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -20,7 +20,5 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
-  when:
-    - previous_build_number == 0 or _build_db_status.rc == 1
-    - database.is_replica is not defined
+  when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 38e82fa4..30fbaa62 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -20,8 +20,6 @@
     config_file: "{{ database.credentials_file }}"
     config_overrides_defaults: true
   run_once: true
-  when:
-    - database.is_replica is not defined
 
 - name: Check if the previous database exists.
   ansible.builtin.shell: "set -o pipefail && mysql --defaults-extra-file={{ database.credentials_file }} -e 'SHOW DATABASES;' | grep -w {{ _mysql_previous_build_database_name }}"
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
index 8d93c118..f8ac6f62 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml
@@ -97,8 +97,10 @@
   when:
     - database.handling is defined
     - database.handling | length > 0
+
 - name: Execute backup tasks.
   ansible.builtin.include_tasks: "deploy-{{ _mysql_handling }}.yml"
+  when: database.is_replica is not defined
 
 # We append privileges instead of replacing,
 # to allow this role to be looped over,
@@ -107,12 +109,16 @@
 # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first.
 - name: Create/update mysql user for TCP connections.
   ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'%';"
-  when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
+  when:
+    - ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
+    - database.is_replica is not defined
   run_once: true
 
 - name: Create/update mysql user for unix socket connections.
   ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'localhost';"
-  when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
+  when:
+    - ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' )
+    - database.is_replica is not defined
   run_once: true
 
 - ansible.builtin.set_fact:
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
index e99a8bda..6040c73a 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml
@@ -19,9 +19,7 @@
   args:
     executable: /bin/bash
   run_once: true
-  when:
-    - previous_build_number > 0
-    - database.is_replica is not defined
+  when: previous_build_number > 0
 
 - name: Delete unpacked dump file.
   ansible.builtin.file:
diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml
index f6076262..9e698c91 100644
--- a/roles/database_backup/database_backup-mysql/tasks/revert.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml
@@ -17,3 +17,4 @@
   when:
     - previous_build_number > 0
     - database_backup.revert
+    - database.is_replica is not defined

From 5b2204e09f73e173f749ec45b09c4d9491a4def0 Mon Sep 17 00:00:00 2001
From: Philip Norton <philipnorton42@gmail.com>
Date: Fri, 8 Mar 2024 10:31:35 +0000
Subject: [PATCH 115/139] 448: Drupal 10 default.settings.php for multi-site MR
 1.x (#449)

* 448: Added setup for the Drupal default.settings.php file, when used in a multi-site context.

* 448: Corrected linting issue.

* 448: Corrected code style issue.

* 448: Added a Drupal default.settings.php file to the templates directory, for use with multi-site Drupal setups.

* 448: Added a .j2 to the end of the default.settings.php template reference.

---------

Co-authored-by: Philip Norton <phil.norton@codeenigma.com>
---
 .../tasks/settings.yml                        |  14 +
 .../templates/default.settings.php.j2         | 883 ++++++++++++++++++
 2 files changed, 897 insertions(+)
 create mode 100644 roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2

diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
index 79e0ad5e..2b07a09b 100644
--- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml
+++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml
@@ -31,3 +31,17 @@
     - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.{{ config_generate_drupal.drupal_settings_file_name }}"
     - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ config_generate_drupal.drupal_settings_file_name }}"
     - "settings.php.j2"
+
+- name: Generates default.settings.php file in any additional multi-site directories.
+  ansible.builtin.template:
+    src: "{{ item }}"
+    dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/default.settings.php"
+  with_first_found:
+    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/default.settings.php"
+    - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.default.settings.php"
+    - "{{ deploy_path }}/{{ webroot }}/sites/default/default.settings.php"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.default.{{ config_generate_drupal.drupal_settings_file_name }}"
+    - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/default.{{ config_generate_drupal.drupal_settings_file_name }}"
+    - "default.settings.php.j2"
+  when:
+    - site.folder != "default"
diff --git a/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2 b/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2
new file mode 100644
index 00000000..63fb2df7
--- /dev/null
+++ b/roles/config_generate/config_generate-drupal8/templates/default.settings.php.j2
@@ -0,0 +1,883 @@
+<?php
+
+// phpcs:ignoreFile
+
+/**
+ * @file
+ * Drupal site-specific configuration file.
+ *
+ * IMPORTANT NOTE:
+ * This file may have been set to read-only by the Drupal installation program.
+ * If you make changes to this file, be sure to protect it again after making
+ * your modifications. Failure to remove write permissions to this file is a
+ * security risk.
+ *
+ * In order to use the selection rules below the multisite aliasing file named
+ * sites/sites.php must be present. Its optional settings will be loaded, and
+ * the aliases in the array $sites will override the default directory rules
+ * below. See sites/example.sites.php for more information about aliases.
+ *
+ * The configuration directory will be discovered by stripping the website's
+ * hostname from left to right and pathname from right to left. The first
+ * configuration file found will be used and any others will be ignored. If no
+ * other configuration file is found then the default configuration file at
+ * 'sites/default' will be used.
+ *
+ * For example, for a fictitious site installed at
+ * https://www.drupal.org:8080/my-site/test/, the 'settings.php' file is searched
+ * for in the following directories:
+ *
+ * - sites/8080.www.drupal.org.my-site.test
+ * - sites/www.drupal.org.my-site.test
+ * - sites/drupal.org.my-site.test
+ * - sites/org.my-site.test
+ *
+ * - sites/8080.www.drupal.org.my-site
+ * - sites/www.drupal.org.my-site
+ * - sites/drupal.org.my-site
+ * - sites/org.my-site
+ *
+ * - sites/8080.www.drupal.org
+ * - sites/www.drupal.org
+ * - sites/drupal.org
+ * - sites/org
+ *
+ * - sites/default
+ *
+ * Note that if you are installing on a non-standard port number, prefix the
+ * hostname with that number. For example,
+ * https://www.drupal.org:8080/my-site/test/ could be loaded from
+ * sites/8080.www.drupal.org.my-site.test/.
+ *
+ * @see example.sites.php
+ * @see \Drupal\Core\DrupalKernel::getSitePath()
+ *
+ * In addition to customizing application settings through variables in
+ * settings.php, you can create a services.yml file in the same directory to
+ * register custom, site-specific service definitions and/or swap out default
+ * implementations with custom ones.
+ */
+
+/**
+ * Database settings:
+ *
+ * The $databases array specifies the database connection or
+ * connections that Drupal may use.  Drupal is able to connect
+ * to multiple databases, including multiple types of databases,
+ * during the same request.
+ *
+ * One example of the simplest connection array is shown below. To use the
+ * sample settings, copy and uncomment the code below between the @code and
+ * @endcode lines and paste it after the $databases declaration. You will need
+ * to replace the database username and password and possibly the host and port
+ * with the appropriate credentials for your database system.
+ *
+ * The next section describes how to customize the $databases array for more
+ * specific needs.
+ *
+ * @code
+ * $databases['default']['default'] = [
+ *   'database' => 'databasename',
+ *   'username' => 'sql_username',
+ *   'password' => 'sql_password',
+ *   'host' => 'localhost',
+ *   'port' => '3306',
+ *   'driver' => 'mysql',
+ *   'prefix' => '',
+ *   'collation' => 'utf8mb4_general_ci',
+ * ];
+ * @endcode
+ */
+$databases = [];
+
+/**
+ * Customizing database settings.
+ *
+ * Many of the values of the $databases array can be customized for your
+ * particular database system. Refer to the sample in the section above as a
+ * starting point.
+ *
+ * The "driver" property indicates what Drupal database driver the
+ * connection should use.  This is usually the same as the name of the
+ * database type, such as mysql or sqlite, but not always.  The other
+ * properties will vary depending on the driver.  For SQLite, you must
+ * specify a database file name in a directory that is writable by the
+ * webserver.  For most other drivers, you must specify a
+ * username, password, host, and database name.
+ *
+ * Drupal core implements drivers for mysql, pgsql, and sqlite. Other drivers
+ * can be provided by contributed or custom modules. To use a contributed or
+ * custom driver, the "namespace" property must be set to the namespace of the
+ * driver. The code in this namespace must be autoloadable prior to connecting
+ * to the database, and therefore, prior to when module root namespaces are
+ * added to the autoloader. To add the driver's namespace to the autoloader,
+ * set the "autoload" property to the PSR-4 base directory of the driver's
+ * namespace. This is optional for projects managed with Composer if the
+ * driver's namespace is in Composer's autoloader.
+ *
+ * For each database, you may optionally specify multiple "target" databases.
+ * A target database allows Drupal to try to send certain queries to a
+ * different database if it can but fall back to the default connection if not.
+ * That is useful for primary/replica replication, as Drupal may try to connect
+ * to a replica server when appropriate and if one is not available will simply
+ * fall back to the single primary server (The terms primary/replica are
+ * traditionally referred to as master/slave in database server documentation).
+ *
+ * The general format for the $databases array is as follows:
+ * @code
+ * $databases['default']['default'] = $info_array;
+ * $databases['default']['replica'][] = $info_array;
+ * $databases['default']['replica'][] = $info_array;
+ * $databases['extra']['default'] = $info_array;
+ * @endcode
+ *
+ * In the above example, $info_array is an array of settings described above.
+ * The first line sets a "default" database that has one primary database
+ * (the second level default).  The second and third lines create an array
+ * of potential replica databases.  Drupal will select one at random for a given
+ * request as needed.  The fourth line creates a new database with a name of
+ * "extra".
+ *
+ * For MySQL, MariaDB or equivalent databases the 'isolation_level' option can
+ * be set. The recommended transaction isolation level for Drupal sites is
+ * 'READ COMMITTED'. The 'REPEATABLE READ' option is supported but can result
+ * in deadlocks, the other two options are 'READ UNCOMMITTED' and 'SERIALIZABLE'.
+ * They are available but not supported; use them at your own risk. For more
+ * info:
+ * https://dev.mysql.com/doc/refman/5.7/en/innodb-transaction-isolation-levels.html
+ *
+ * On your settings.php, change the isolation level:
+ * @code
+ * $databases['default']['default']['init_commands'] = [
+ *   'isolation_level' => 'SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED',
+ * ];
+ * @endcode
+ *
+ * You can optionally set a prefix for all database table names by using the
+ * 'prefix' setting. If a prefix is specified, the table name will be prepended
+ * with its value. Be sure to use valid database characters only, usually
+ * alphanumeric and underscore. If no prefix is desired, do not set the 'prefix'
+ * key or set its value to an empty string ''.
+ *
+ * For example, to have all database table prefixed with 'main_', set:
+ * @code
+ *   'prefix' => 'main_',
+ * @endcode
+ *
+ * Advanced users can add or override initial commands to execute when
+ * connecting to the database server, as well as PDO connection settings. For
+ * example, to enable MySQL SELECT queries to exceed the max_join_size system
+ * variable, and to reduce the database connection timeout to 5 seconds:
+ * @code
+ * $databases['default']['default'] = [
+ *   'init_commands' => [
+ *     'big_selects' => 'SET SQL_BIG_SELECTS=1',
+ *   ],
+ *   'pdo' => [
+ *     PDO::ATTR_TIMEOUT => 5,
+ *   ],
+ * ];
+ * @endcode
+ *
+ * WARNING: The above defaults are designed for database portability. Changing
+ * them may cause unexpected behavior, including potential data loss. See
+ * https://www.drupal.org/developing/api/database/configuration for more
+ * information on these defaults and the potential issues.
+ *
+ * More details can be found in the constructor methods for each driver:
+ * - \Drupal\mysql\Driver\Database\mysql\Connection::__construct()
+ * - \Drupal\pgsql\Driver\Database\pgsql\Connection::__construct()
+ * - \Drupal\sqlite\Driver\Database\sqlite\Connection::__construct()
+ *
+ * Sample Database configuration format for PostgreSQL (pgsql):
+ * @code
+ *   $databases['default']['default'] = [
+ *     'driver' => 'pgsql',
+ *     'database' => 'databasename',
+ *     'username' => 'sql_username',
+ *     'password' => 'sql_password',
+ *     'host' => 'localhost',
+ *     'prefix' => '',
+ *   ];
+ * @endcode
+ *
+ * Sample Database configuration format for SQLite (sqlite):
+ * @code
+ *   $databases['default']['default'] = [
+ *     'driver' => 'sqlite',
+ *     'database' => '/path/to/database_filename',
+ *   ];
+ * @endcode
+ *
+ * Sample Database configuration format for a driver in a contributed module:
+ * @code
+ *   $databases['default']['default'] = [
+ *     'driver' => 'my_driver',
+ *     'namespace' => 'Drupal\my_module\Driver\Database\my_driver',
+ *     'autoload' => 'modules/my_module/src/Driver/Database/my_driver/',
+ *     'database' => 'databasename',
+ *     'username' => 'sql_username',
+ *     'password' => 'sql_password',
+ *     'host' => 'localhost',
+ *     'prefix' => '',
+ *   ];
+ * @endcode
+ *
+ * Sample Database configuration format for a driver that is extending another
+ * database driver.
+ * @code
+ *   $databases['default']['default'] = [
+ *     'driver' => 'my_driver',
+ *     'namespace' => 'Drupal\my_module\Driver\Database\my_driver',
+ *     'autoload' => 'modules/my_module/src/Driver/Database/my_driver/',
+ *     'database' => 'databasename',
+ *     'username' => 'sql_username',
+ *     'password' => 'sql_password',
+ *     'host' => 'localhost',
+ *     'prefix' => '',
+ *     'dependencies' => [
+ *       'parent_module' => [
+ *         'namespace' => 'Drupal\parent_module',
+ *         'autoload' => 'core/modules/parent_module/src/',
+ *       ],
+ *     ],
+ *   ];
+ * @endcode
+ */
+
+/**
+ * Location of the site configuration files.
+ *
+ * The $settings['config_sync_directory'] specifies the location of file system
+ * directory used for syncing configuration data. On install, the directory is
+ * created. This is used for configuration imports.
+ *
+ * The default location for this directory is inside a randomly-named
+ * directory in the public files path. The setting below allows you to set
+ * its location.
+ */
+# $settings['config_sync_directory'] = '/directory/outside/webroot';
+
+/**
+ * Settings:
+ *
+ * $settings contains environment-specific configuration, such as the files
+ * directory and reverse proxy address, and temporary configuration, such as
+ * security overrides.
+ *
+ * @see \Drupal\Core\Site\Settings::get()
+ */
+
+/**
+ * Salt for one-time login links, cancel links, form tokens, etc.
+ *
+ * This variable will be set to a random value by the installer. All one-time
+ * login links will be invalidated if the value is changed. Note that if your
+ * site is deployed on a cluster of web servers, you must ensure that this
+ * variable has the same value on each server.
+ *
+ * For enhanced security, you may set this variable to the contents of a file
+ * outside your document root, and vary the value across environments (like
+ * production and development); you should also ensure that this file is not
+ * stored with backups of your database.
+ *
+ * Example:
+ * @code
+ *   $settings['hash_salt'] = file_get_contents('/home/example/salt.txt');
+ * @endcode
+ */
+$settings['hash_salt'] = '';
+
+/**
+ * Deployment identifier.
+ *
+ * Drupal's dependency injection container will be automatically invalidated and
+ * rebuilt when the Drupal core version changes. When updating contributed or
+ * custom code that changes the container, changing this identifier will also
+ * allow the container to be invalidated as soon as code is deployed.
+ */
+# $settings['deployment_identifier'] = \Drupal::VERSION;
+
+/**
+ * Access control for update.php script.
+ *
+ * If you are updating your Drupal installation using the update.php script but
+ * are not logged in using either an account with the "Administer software
+ * updates" permission or the site maintenance account (the account that was
+ * created during installation), you will need to modify the access check
+ * statement below. Change the FALSE to a TRUE to disable the access check.
+ * After finishing the upgrade, be sure to open this file again and change the
+ * TRUE back to a FALSE!
+ */
+$settings['update_free_access'] = FALSE;
+
+/**
+ * Fallback to HTTP for Update Manager and for fetching security advisories.
+ *
+ * If your site fails to connect to updates.drupal.org over HTTPS (either when
+ * fetching data on available updates, or when fetching the feed of critical
+ * security announcements), you may uncomment this setting and set it to TRUE to
+ * allow an insecure fallback to HTTP. Note that doing so will open your site up
+ * to a potential man-in-the-middle attack. You should instead attempt to
+ * resolve the issues before enabling this option.
+ * @see https://www.drupal.org/docs/system-requirements/php-requirements#openssl
+ * @see https://en.wikipedia.org/wiki/Man-in-the-middle_attack
+ * @see \Drupal\update\UpdateFetcher
+ * @see \Drupal\system\SecurityAdvisories\SecurityAdvisoriesFetcher
+ */
+# $settings['update_fetch_with_http_fallback'] = TRUE;
+
+/**
+ * External access proxy settings:
+ *
+ * If your site must access the Internet via a web proxy then you can enter the
+ * proxy settings here. Set the full URL of the proxy, including the port, in
+ * variables:
+ * - $settings['http_client_config']['proxy']['http']: The proxy URL for HTTP
+ *   requests.
+ * - $settings['http_client_config']['proxy']['https']: The proxy URL for HTTPS
+ *   requests.
+ * You can pass in the user name and password for basic authentication in the
+ * URLs in these settings.
+ *
+ * You can also define an array of host names that can be accessed directly,
+ * bypassing the proxy, in $settings['http_client_config']['proxy']['no'].
+ */
+# $settings['http_client_config']['proxy']['http'] = 'http://proxy_user:proxy_pass@example.com:8080';
+# $settings['http_client_config']['proxy']['https'] = 'http://proxy_user:proxy_pass@example.com:8080';
+# $settings['http_client_config']['proxy']['no'] = ['127.0.0.1', 'localhost'];
+
+/**
+ * Reverse Proxy Configuration:
+ *
+ * Reverse proxy servers are often used to enhance the performance
+ * of heavily visited sites and may also provide other site caching,
+ * security, or encryption benefits. In an environment where Drupal
+ * is behind a reverse proxy, the real IP address of the client should
+ * be determined such that the correct client IP address is available
+ * to Drupal's logging, statistics, and access management systems. In
+ * the most simple scenario, the proxy server will add an
+ * X-Forwarded-For header to the request that contains the client IP
+ * address. However, HTTP headers are vulnerable to spoofing, where a
+ * malicious client could bypass restrictions by setting the
+ * X-Forwarded-For header directly. Therefore, Drupal's proxy
+ * configuration requires the IP addresses of all remote proxies to be
+ * specified in $settings['reverse_proxy_addresses'] to work correctly.
+ *
+ * Enable this setting to get Drupal to determine the client IP from the
+ * X-Forwarded-For header. If you are unsure about this setting, do not have a
+ * reverse proxy, or Drupal operates in a shared hosting environment, this
+ * setting should remain commented out.
+ *
+ * In order for this setting to be used you must specify every possible
+ * reverse proxy IP address in $settings['reverse_proxy_addresses'].
+ * If a complete list of reverse proxies is not available in your
+ * environment (for example, if you use a CDN) you may set the
+ * $_SERVER['REMOTE_ADDR'] variable directly in settings.php.
+ * Be aware, however, that it is likely that this would allow IP
+ * address spoofing unless more advanced precautions are taken.
+ */
+# $settings['reverse_proxy'] = TRUE;
+
+/**
+ * Reverse proxy addresses.
+ *
+ * Specify every reverse proxy IP address in your environment, as an array of
+ * IPv4/IPv6 addresses or subnets in CIDR notation. This setting is required if
+ * $settings['reverse_proxy'] is TRUE.
+ */
+# $settings['reverse_proxy_addresses'] = ['a.b.c.d', 'e.f.g.h/24', ...];
+
+/**
+ * Reverse proxy trusted headers.
+ *
+ * Sets which headers to trust from your reverse proxy.
+ *
+ * Common values are:
+ * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR
+ * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST
+ * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT
+ * - \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO
+ * - \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED
+ *
+ * Note the default value of
+ * @code
+ * \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO | \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED
+ * @endcode
+ * is not secure by default. The value should be set to only the specific
+ * headers the reverse proxy uses. For example:
+ * @code
+ * \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO
+ * @endcode
+ * This would trust the following headers:
+ * - X_FORWARDED_FOR
+ * - X_FORWARDED_HOST
+ * - X_FORWARDED_PROTO
+ * - X_FORWARDED_PORT
+ *
+ * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR
+ * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST
+ * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT
+ * @see \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO
+ * @see \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED
+ * @see \Symfony\Component\HttpFoundation\Request::setTrustedProxies
+ */
+# $settings['reverse_proxy_trusted_headers'] = \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_FOR | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_HOST | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PORT | \Symfony\Component\HttpFoundation\Request::HEADER_X_FORWARDED_PROTO | \Symfony\Component\HttpFoundation\Request::HEADER_FORWARDED;
+
+
+/**
+ * Page caching:
+ *
+ * By default, Drupal sends a "Vary: Cookie" HTTP header for anonymous page
+ * views. This tells a HTTP proxy that it may return a page from its local
+ * cache without contacting the web server, if the user sends the same Cookie
+ * header as the user who originally requested the cached page. Without "Vary:
+ * Cookie", authenticated users would also be served the anonymous page from
+ * the cache. If the site has mostly anonymous users except a few known
+ * editors/administrators, the Vary header can be omitted. This allows for
+ * better caching in HTTP proxies (including reverse proxies), i.e. even if
+ * clients send different cookies, they still get content served from the cache.
+ * However, authenticated users should access the site directly (i.e. not use an
+ * HTTP proxy, and bypass the reverse proxy if one is used) in order to avoid
+ * getting cached pages from the proxy.
+ */
+# $settings['omit_vary_cookie'] = TRUE;
+
+
+/**
+ * Cache TTL for client error (4xx) responses.
+ *
+ * Items cached per-URL tend to result in a large number of cache items, and
+ * this can be problematic on 404 pages which by their nature are unbounded. A
+ * fixed TTL can be set for these items, defaulting to one hour, so that cache
+ * backends which do not support LRU can purge older entries. To disable caching
+ * of client error responses set the value to 0. Currently applies only to
+ * page_cache module.
+ */
+# $settings['cache_ttl_4xx'] = 3600;
+
+/**
+ * Expiration of cached forms.
+ *
+ * Drupal's Form API stores details of forms in a cache and these entries are
+ * kept for at least 6 hours by default. Expired entries are cleared by cron.
+ *
+ * @see \Drupal\Core\Form\FormCache::setCache()
+ */
+# $settings['form_cache_expiration'] = 21600;
+
+/**
+ * Class Loader.
+ *
+ * If the APCu extension is detected, the classloader will be optimized to use
+ * it. Set to FALSE to disable this.
+ *
+ * @see https://getcomposer.org/doc/articles/autoloader-optimization.md
+ */
+# $settings['class_loader_auto_detect'] = FALSE;
+
+/**
+ * Authorized file system operations:
+ *
+ * The Update Manager module included with Drupal provides a mechanism for
+ * site administrators to securely install missing updates for the site
+ * directly through the web user interface. On securely-configured servers,
+ * the Update manager will require the administrator to provide SSH or FTP
+ * credentials before allowing the installation to proceed; this allows the
+ * site to update the new files as the user who owns all the Drupal files,
+ * instead of as the user the webserver is running as. On servers where the
+ * webserver user is itself the owner of the Drupal files, the administrator
+ * will not be prompted for SSH or FTP credentials (note that these server
+ * setups are common on shared hosting, but are inherently insecure).
+ *
+ * Some sites might wish to disable the above functionality, and only update
+ * the code directly via SSH or FTP themselves. This setting completely
+ * disables all functionality related to these authorized file operations.
+ *
+ * @see https://www.drupal.org/node/244924
+ *
+ * Remove the leading hash signs to disable.
+ */
+# $settings['allow_authorize_operations'] = FALSE;
+
+/**
+ * Default mode for directories and files written by Drupal.
+ *
+ * Value should be in PHP Octal Notation, with leading zero.
+ */
+# $settings['file_chmod_directory'] = 0775;
+# $settings['file_chmod_file'] = 0664;
+
+/**
+ * Optimized assets path:
+ *
+ * A local file system path where optimized assets will be stored. This directory
+ * must exist and be writable by Drupal. This directory must be relative to
+ * the Drupal installation directory and be accessible over the web.
+ */
+# $settings['file_assets_path'] = 'sites/default/files';
+
+/**
+ * Public file base URL:
+ *
+ * An alternative base URL to be used for serving public files. This must
+ * include any leading directory path.
+ *
+ * A different value from the domain used by Drupal to be used for accessing
+ * public files. This can be used for a simple CDN integration, or to improve
+ * security by serving user-uploaded files from a different domain or subdomain
+ * pointing to the same server. Do not include a trailing slash.
+ */
+# $settings['file_public_base_url'] = 'http://downloads.example.com/files';
+
+/**
+ * Public file path:
+ *
+ * A local file system path where public files will be stored. This directory
+ * must exist and be writable by Drupal. This directory must be relative to
+ * the Drupal installation directory and be accessible over the web.
+ */
+# $settings['file_public_path'] = 'sites/default/files';
+
+/**
+ * Additional public file schemes:
+ *
+ * Public schemes are URI schemes that allow download access to all users for
+ * all files within that scheme.
+ *
+ * The "public" scheme is always public, and the "private" scheme is always
+ * private, but other schemes, such as "https", "s3", "example", or others,
+ * can be either public or private depending on the site. By default, they're
+ * private, and access to individual files is controlled via
+ * hook_file_download().
+ *
+ * Typically, if a scheme should be public, a module makes it public by
+ * implementing hook_file_download(), and granting access to all users for all
+ * files. This could be either the same module that provides the stream wrapper
+ * for the scheme, or a different module that decides to make the scheme
+ * public. However, in cases where a site needs to make a scheme public, but
+ * is unable to add code in a module to do so, the scheme may be added to this
+ * variable, the result of which is that system_file_download() grants public
+ * access to all files within that scheme.
+ */
+# $settings['file_additional_public_schemes'] = ['example'];
+
+/**
+ * File schemes whose paths should not be normalized:
+ *
+ * Normally, Drupal normalizes '/./' and '/../' segments in file URIs in order
+ * to prevent unintended file access. For example, 'private://css/../image.png'
+ * is normalized to 'private://image.png' before checking access to the file.
+ *
+ * On Windows, Drupal also replaces '\' with '/' in URIs for the local
+ * filesystem.
+ *
+ * If file URIs with one or more scheme should not be normalized like this, then
+ * list the schemes here. For example, if 'porcelain://china/./plate.png' should
+ * not be normalized to 'porcelain://china/plate.png', then add 'porcelain' to
+ * this array. In this case, make sure that the module providing the 'porcelain'
+ * scheme does not allow unintended file access when using '/../' to move up the
+ * directory tree.
+ */
+# $settings['file_sa_core_2023_005_schemes'] = ['porcelain'];
+
+/**
+ * Configuration for phpinfo() admin status report.
+ *
+ * Drupal's admin UI includes a report at admin/reports/status/php which shows
+ * the output of phpinfo(). The full output can contain sensitive information
+ * so by default Drupal removes some sections.
+ *
+ * This behavior can be configured by setting this variable to a different
+ * value corresponding to the flags parameter of phpinfo().
+ *
+ * If you need to expose more information in the report - for example to debug a
+ * problem - consider doing so temporarily.
+ *
+ * @see https://www.php.net/manual/function.phpinfo.php
+ */
+# $settings['sa_core_2023_004_phpinfo_flags'] = ~ (INFO_VARIABLES | INFO_ENVIRONMENT);
+
+/**
+ * Private file path:
+ *
+ * A local file system path where private files will be stored. This directory
+ * must be absolute, outside of the Drupal installation directory and not
+ * accessible over the web.
+ *
+ * Note: Caches need to be cleared when this value is changed to make the
+ * private:// stream wrapper available to the system.
+ *
+ * See https://www.drupal.org/documentation/modules/file for more information
+ * about securing private files.
+ */
+# $settings['file_private_path'] = '';
+
+/**
+ * Temporary file path:
+ *
+ * A local file system path where temporary files will be stored. This directory
+ * must be absolute, outside of the Drupal installation directory and not
+ * accessible over the web.
+ *
+ * If this is not set, the default for the operating system will be used.
+ *
+ * @see \Drupal\Component\FileSystem\FileSystem::getOsTemporaryDirectory()
+ */
+# $settings['file_temp_path'] = '/tmp';
+
+/**
+ * Session write interval:
+ *
+ * Set the minimum interval between each session write to database.
+ * For performance reasons it defaults to 180.
+ */
+# $settings['session_write_interval'] = 180;
+
+/**
+ * String overrides:
+ *
+ * To override specific strings on your site with or without enabling the Locale
+ * module, add an entry to this list. This functionality allows you to change
+ * a small number of your site's default English language interface strings.
+ *
+ * Remove the leading hash signs to enable.
+ *
+ * The "en" part of the variable name, is dynamic and can be any langcode of
+ * any added language. (eg locale_custom_strings_de for german).
+ */
+# $settings['locale_custom_strings_en'][''] = [
+#   'Home' => 'Front page',
+#   '@count min' => '@count minutes',
+# ];
+
+/**
+ * A custom theme for the offline page:
+ *
+ * This applies when the site is explicitly set to maintenance mode through the
+ * administration page or when the database is inactive due to an error.
+ * The template file should also be copied into the theme. It is located inside
+ * 'core/modules/system/templates/maintenance-page.html.twig'.
+ *
+ * Note: This setting does not apply to installation and update pages.
+ */
+# $settings['maintenance_theme'] = 'claro';
+
+/**
+ * PHP settings:
+ *
+ * To see what PHP settings are possible, including whether they can be set at
+ * runtime (by using ini_set()), read the PHP documentation:
+ * http://php.net/manual/ini.list.php
+ * See \Drupal\Core\DrupalKernel::bootEnvironment() for required runtime
+ * settings and the .htaccess file for non-runtime settings.
+ * Settings defined there should not be duplicated here so as to avoid conflict
+ * issues.
+ */
+
+/**
+ * If you encounter a situation where users post a large amount of text, and
+ * the result is stripped out upon viewing but can still be edited, Drupal's
+ * output filter may not have sufficient memory to process it.  If you
+ * experience this issue, you may wish to uncomment the following two lines
+ * and increase the limits of these variables.  For more information, see
+ * http://php.net/manual/pcre.configuration.php.
+ */
+# ini_set('pcre.backtrack_limit', 200000);
+# ini_set('pcre.recursion_limit', 200000);
+
+/**
+ * Configuration overrides.
+ *
+ * To globally override specific configuration values for this site,
+ * set them here. You usually don't need to use this feature. This is
+ * useful in a configuration file for a vhost or directory, rather than
+ * the default settings.php.
+ *
+ * Note that any values you provide in these variable overrides will not be
+ * viewable from the Drupal administration interface. The administration
+ * interface displays the values stored in configuration so that you can stage
+ * changes to other environments that don't have the overrides.
+ *
+ * There are particular configuration values that are risky to override. For
+ * example, overriding the list of installed modules in 'core.extension' is not
+ * supported as module install or uninstall has not occurred. Other examples
+ * include field storage configuration, because it has effects on database
+ * structure, and 'core.menu.static_menu_link_overrides' since this is cached in
+ * a way that is not config override aware. Also, note that changing
+ * configuration values in settings.php will not fire any of the configuration
+ * change events.
+ */
+# $config['system.site']['name'] = 'My Drupal site';
+# $config['user.settings']['anonymous'] = 'Visitor';
+
+/**
+ * Load services definition file.
+ */
+$settings['container_yamls'][] = $app_root . '/' . $site_path . '/services.yml';
+
+/**
+ * Override the default service container class.
+ *
+ * This is useful for example to trace the service container for performance
+ * tracking purposes, for testing a service container with an error condition or
+ * to test a service container that throws an exception.
+ */
+# $settings['container_base_class'] = '\Drupal\Core\DependencyInjection\Container';
+
+/**
+ * Override the default yaml parser class.
+ *
+ * Provide a fully qualified class name here if you would like to provide an
+ * alternate implementation YAML parser. The class must implement the
+ * \Drupal\Component\Serialization\SerializationInterface interface.
+ */
+# $settings['yaml_parser_class'] = NULL;
+
+/**
+ * Trusted host configuration.
+ *
+ * Drupal core can use the Symfony trusted host mechanism to prevent HTTP Host
+ * header spoofing.
+ *
+ * To enable the trusted host mechanism, you enable your allowable hosts
+ * in $settings['trusted_host_patterns']. This should be an array of regular
+ * expression patterns, without delimiters, representing the hosts you would
+ * like to allow.
+ *
+ * For example:
+ * @code
+ * $settings['trusted_host_patterns'] = [
+ *   '^www\.example\.com$',
+ * ];
+ * @endcode
+ * will allow the site to only run from www.example.com.
+ *
+ * If you are running multisite, or if you are running your site from
+ * different domain names (eg, you don't redirect http://www.example.com to
+ * http://example.com), you should specify all of the host patterns that are
+ * allowed by your site.
+ *
+ * For example:
+ * @code
+ * $settings['trusted_host_patterns'] = [
+ *   '^example\.com$',
+ *   '^.+\.example\.com$',
+ *   '^example\.org$',
+ *   '^.+\.example\.org$',
+ * ];
+ * @endcode
+ * will allow the site to run off of all variants of example.com and
+ * example.org, with all subdomains included.
+ *
+ * @see https://www.drupal.org/docs/installing-drupal/trusted-host-settings
+ */
+# $settings['trusted_host_patterns'] = [];
+
+/**
+ * The default list of directories that will be ignored by Drupal's file API.
+ *
+ * By default ignore node_modules and bower_components folders to avoid issues
+ * with common frontend tools and recursive scanning of directories looking for
+ * extensions.
+ *
+ * @see \Drupal\Core\File\FileSystemInterface::scanDirectory()
+ * @see \Drupal\Core\Extension\ExtensionDiscovery::scanDirectory()
+ */
+$settings['file_scan_ignore_directories'] = [
+  'node_modules',
+  'bower_components',
+];
+
+/**
+ * The default number of entities to update in a batch process.
+ *
+ * This is used by update and post-update functions that need to go through and
+ * change all the entities on a site, so it is useful to increase this number
+ * if your hosting configuration (i.e. RAM allocation, CPU speed) allows for a
+ * larger number of entities to be processed in a single batch run.
+ */
+$settings['entity_update_batch_size'] = 50;
+
+/**
+ * Entity update backup.
+ *
+ * This is used to inform the entity storage handler that the backup tables as
+ * well as the original entity type and field storage definitions should be
+ * retained after a successful entity update process.
+ */
+$settings['entity_update_backup'] = TRUE;
+
+/**
+ * Node migration type.
+ *
+ * This is used to force the migration system to use the classic node migrations
+ * instead of the default complete node migrations. The migration system will
+ * use the classic node migration only if there are existing migrate_map tables
+ * for the classic node migrations and they contain data. These tables may not
+ * exist if you are developing custom migrations and do not want to use the
+ * complete node migrations. Set this to TRUE to force the use of the classic
+ * node migrations.
+ */
+$settings['migrate_node_migrate_type_classic'] = FALSE;
+
+/**
+ * The default settings for migration sources.
+ *
+ * These settings are used as the default settings on the Credential form at
+ * /upgrade/credentials.
+ *
+ * - migrate_source_version - The version of the source database. This can be
+ *   '6' or '7'. Defaults to '7'.
+ * - migrate_source_connection - The key in the $databases array for the source
+ *   site.
+ * - migrate_file_public_path - The location of the source Drupal 6 or Drupal 7
+ *   public files. This can be a local file directory containing the source
+ *   Drupal 6 or Drupal 7 site (e.g /var/www/docroot), or the site address
+ *   (e.g http://example.com).
+ * - migrate_file_private_path - The location of the source Drupal 7 private
+ *   files. This can be a local file directory containing the source Drupal 7
+ *   site (e.g /var/www/docroot), or empty to use the same value as Public
+ *   files directory.
+ *
+ * Sample configuration for a drupal 6 source site with the source files in a
+ * local directory.
+ *
+ * @code
+ * $settings['migrate_source_version'] = '6';
+ * $settings['migrate_source_connection'] = 'migrate';
+ * $settings['migrate_file_public_path'] = '/var/www/drupal6';
+ * @endcode
+ *
+ * Sample configuration for a drupal 7 source site with public source files on
+ * the source site and the private files in a local directory.
+ *
+ * @code
+ * $settings['migrate_source_version'] = '7';
+ * $settings['migrate_source_connection'] = 'migrate';
+ * $settings['migrate_file_public_path'] = 'https://drupal7.com';
+ * $settings['migrate_file_private_path'] = '/var/www/drupal7';
+ * @endcode
+ */
+# $settings['migrate_source_connection'] = '';
+# $settings['migrate_source_version'] = '';
+# $settings['migrate_file_public_path'] = '';
+# $settings['migrate_file_private_path'] = '';
+
+/**
+ * Load local development override configuration, if available.
+ *
+ * Create a settings.local.php file to override variables on secondary (staging,
+ * development, etc.) installations of this site.
+ *
+ * Typical uses of settings.local.php include:
+ * - Disabling caching.
+ * - Disabling JavaScript/CSS compression.
+ * - Rerouting outgoing emails.
+ *
+ * Keep this code block at the end of this file to take full effect.
+ */
+#
+# if (file_exists($app_root . '/' . $site_path . '/settings.local.php')) {
+#   include $app_root . '/' . $site_path . '/settings.local.php';
+# }

From 87779870a7c99932ae43c1c4a764685331246ad4 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 4 Apr 2024 18:31:27 +0200
Subject: [PATCH 116/139] New StatusCake check handling role. (#454)

* New StatusCake check handling role.

* Updating maintenance mode defaults and options.

* Following StatusCake task naming conventions.
---
 docs/_Sidebar.md                              |  1 +
 docs/roles/maintenance_mode.md                | 26 +++++++++++++++++++
 roles/maintenance_mode/README.md              | 26 +++++++++++++++++++
 roles/maintenance_mode/defaults/main.yml      | 10 +++----
 .../maintenance_mode-nginx/defaults/main.yml  |  2 +-
 .../defaults/main.yml                         |  5 ++++
 .../tasks/main.yml                            |  8 ++++++
 .../tasks/offline.yml                         | 17 ++++++++++++
 .../tasks/online.yml                          | 17 ++++++++++++
 9 files changed, 106 insertions(+), 6 deletions(-)
 create mode 100644 docs/roles/maintenance_mode.md
 create mode 100644 roles/maintenance_mode/README.md
 create mode 100644 roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
 create mode 100644 roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
 create mode 100644 roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
 create mode 100644 roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index 3d47469a..a58673b3 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -34,6 +34,7 @@
      - [Deploy container](/roles/deploy_container)
      - [Init](/roles/_init)
      - [LHCI run](/roles/lhci_run)
+     - [Maintenance Mode](/roles/maintenance_mode)
      - ["Meta"](/roles/_meta)
        - [Drupal 7](/roles/_meta/deploy-drupal7)
        - [Drupal 8](/roles/_meta/deploy-drupal8)
diff --git a/docs/roles/maintenance_mode.md b/docs/roles/maintenance_mode.md
new file mode 100644
index 00000000..ec9d640c
--- /dev/null
+++ b/docs/roles/maintenance_mode.md
@@ -0,0 +1,26 @@
+# Maintenance Mode
+This role and its sub-roles handle various methods for putting applications into an offline maintenance state. See each sub-role's own defaults for additional variables.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+# Puts site(s) offline.
+maintenance_mode:
+  # Where do we apply maintenance - can be invoked more than one time for different purposes.
+  # - nginx: serves a static maintenance page
+  # - drupal-core: application level
+  # - statuscake: disables a StatusCake check
+  # @todo - haproxy: serves a static maintenance page.
+  # @todo - drupal-read-only: application level via readonly module.
+  mode: "nginx"
+  # - offline: puts the site offline
+  # @todo - restricted: put the site offline except for whitelist (nginx/haproxy only)
+  # - online: brings the site back online.
+  operation: "offline"
+```
+
+<!--ENDROLEVARS-->
diff --git a/roles/maintenance_mode/README.md b/roles/maintenance_mode/README.md
new file mode 100644
index 00000000..ec9d640c
--- /dev/null
+++ b/roles/maintenance_mode/README.md
@@ -0,0 +1,26 @@
+# Maintenance Mode
+This role and its sub-roles handle various methods for putting applications into an offline maintenance state. See each sub-role's own defaults for additional variables.
+
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+# Puts site(s) offline.
+maintenance_mode:
+  # Where do we apply maintenance - can be invoked more than one time for different purposes.
+  # - nginx: serves a static maintenance page
+  # - drupal-core: application level
+  # - statuscake: disables a StatusCake check
+  # @todo - haproxy: serves a static maintenance page.
+  # @todo - drupal-read-only: application level via readonly module.
+  mode: "nginx"
+  # - offline: puts the site offline
+  # @todo - restricted: put the site offline except for whitelist (nginx/haproxy only)
+  # - online: brings the site back online.
+  operation: "offline"
+```
+
+<!--ENDROLEVARS-->
diff --git a/roles/maintenance_mode/defaults/main.yml b/roles/maintenance_mode/defaults/main.yml
index ab895227..15c050e9 100644
--- a/roles/maintenance_mode/defaults/main.yml
+++ b/roles/maintenance_mode/defaults/main.yml
@@ -1,12 +1,12 @@
 ---
 # Puts site(s) offline.
 maintenance_mode:
-  # What level do we operate.
-  # - nginx: serves a static maintenance page.
-  # - drupal_core: application level
-  # - mautic: application level
+  # Where do we apply maintenance - can be invoked more than one time for different purposes.
+  # - nginx: serves a static maintenance page
+  # - drupal-core: application level
+  # - statuscake: disables a StatusCake check
   # @todo - haproxy: serves a static maintenance page.
-  # @todo - drupal_read_only: application level via readonly module.
+  # @todo - drupal-read-only: application level via readonly module.
   mode: "nginx"
   # - offline: puts the site offline
   # @todo - restricted: put the site offline except for whitelist (nginx/haproxy only)
diff --git a/roles/maintenance_mode/maintenance_mode-nginx/defaults/main.yml b/roles/maintenance_mode/maintenance_mode-nginx/defaults/main.yml
index 94e4737a..193e6a4d 100644
--- a/roles/maintenance_mode/maintenance_mode-nginx/defaults/main.yml
+++ b/roles/maintenance_mode/maintenance_mode-nginx/defaults/main.yml
@@ -2,4 +2,4 @@
 maintenance_mode_nginx:
   domains:
     - server_name: "{{ domain_name }}"
-      maintenance_page: "index.html"
\ No newline at end of file
+      maintenance_page: "index.html"
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml b/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
new file mode 100644
index 00000000..097a55bd
--- /dev/null
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+maintenance_mode_statuscake:
+  domains:
+    - statuscake_check_id: "" # Your StatusCake Test ID
+      statuscake_api_key: "abcdef123456" # Your StatusCake API key - create one on your user page - https://app.statuscake.com/User.php
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
new file mode 100644
index 00000000..15b5cac8
--- /dev/null
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- ansible.builtin.include_tasks: "{{ maintenance_mode.operation }}.yml"
+  with_items: "{{ maintenance_mode_statuscake.domains }}"
+  loop_control:
+    loop_var: domain
+  run_once: true
+  when:
+    - maintenance_mode_statuscake.domains | length > 0
\ No newline at end of file
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
new file mode 100644
index 00000000..3cd5a186
--- /dev/null
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
@@ -0,0 +1,17 @@
+---
+- name: Pause StatusCake check.
+  ansible.builtin.uri:
+    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
+    method: PUT
+    body:
+      paused: true
+    headers:
+      Authorization: "Bearer {{ domain.statuscake_api_key }}"
+    status_code:
+      - 200
+      - 204
+  when:
+    - domain.statuscake_check_id is defined
+    - domain.statuscake_check_id | length > 0
+    - domain.statuscake_api_key is defined
+    - domain.statuscake_api_key | length > 0
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
new file mode 100644
index 00000000..63663630
--- /dev/null
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
@@ -0,0 +1,17 @@
+---
+- name: Resume StatusCake check.
+  ansible.builtin.uri:
+    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
+    method: PUT
+    body:
+      paused: false
+    headers:
+      Authorization: "Bearer {{ domain.statuscake_api_key }}"
+    status_code:
+      - 200
+      - 204
+  when:
+    - domain.statuscake_check_id is defined
+    - domain.statuscake_check_id | length > 0
+    - domain.statuscake_api_key is defined
+    - domain.statuscake_api_key | length > 0

From 4f52908828030a8838a6a760f183a1d9db749e04 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 4 Apr 2024 19:40:24 +0200
Subject: [PATCH 117/139] Statuscake maintenance role pr 1.x (#456)

* New StatusCake check handling role.

* Updating maintenance mode defaults and options.

* Following StatusCake task naming conventions.

* Missed the body_format param in StatusCake URI call.
---
 .../maintenance_mode-statuscake/defaults/main.yml               | 2 +-
 .../maintenance_mode-statuscake/tasks/offline.yml               | 1 +
 .../maintenance_mode-statuscake/tasks/online.yml                | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml b/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
index 097a55bd..57240d01 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/defaults/main.yml
@@ -1,5 +1,5 @@
 ---
 maintenance_mode_statuscake:
   domains:
-    - statuscake_check_id: "" # Your StatusCake Test ID
+    - statuscake_check_id: "" # Your StatusCake Test ID - must always be surrounded by double-quotes
       statuscake_api_key: "abcdef123456" # Your StatusCake API key - create one on your user page - https://app.statuscake.com/User.php
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
index 3cd5a186..62081308 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
@@ -3,6 +3,7 @@
   ansible.builtin.uri:
     url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
     method: PUT
+    body_format: form-urlencoded
     body:
       paused: true
     headers:
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
index 63663630..5818ed33 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
@@ -3,6 +3,7 @@
   ansible.builtin.uri:
     url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
     method: PUT
+    body_format: form-urlencoded
     body:
       paused: false
     headers:

From 95dc9608b473e120ea0fcea0e6625874fef05d58 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 5 Apr 2024 17:16:03 +0200
Subject: [PATCH 118/139] Statuscake maintenance role pr 1.x (#458)

* New StatusCake check handling role.

* Updating maintenance mode defaults and options.

* Following StatusCake task naming conventions.

* Missed the body_format param in StatusCake URI call.

* Ensuring we pass the test ID as a string to the StatusCake URI call.

* Fixing namespaces.

* Stashing the original deploy_path in initial_deploy_path for squashfs builds.

* Adding a double online step for Drupal builds using SquashFS so both the new site and the currently live site come out of maintenance mode.

* We need the path to the previous build for maintenance mode online.

* Changing order of tasks so previous_build_number is set earlier.

* We also need a previous_drush_bin.
---
 roles/_init/tasks/drupal8.yml                 |  4 +++
 roles/_init/tasks/main.yml                    | 14 +++++----
 .../tasks/online.yml                          | 29 +++++++++++++++++--
 .../maintenance_mode-nginx/tasks/main.yml     |  4 +--
 .../maintenance_mode-nginx/tasks/offline.yml  |  4 +--
 .../maintenance_mode-nginx/tasks/online.yml   |  4 +--
 .../tasks/main.yml                            |  2 +-
 .../tasks/offline.yml                         |  2 +-
 .../tasks/online.yml                          |  2 +-
 roles/maintenance_mode/tasks/main.yml         |  2 +-
 10 files changed, 50 insertions(+), 17 deletions(-)

diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml
index 7c64719e..eb3bdae9 100644
--- a/roles/_init/tasks/drupal8.yml
+++ b/roles/_init/tasks/drupal8.yml
@@ -3,6 +3,10 @@
   set_fact:
     drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}"
 
+- name: Define path to drush for previous build.
+  set_fact:
+    previous_drush_bin: "{{ previous_drush_bin | default('{{ previous_deploy_path }}/vendor/bin/drush') }}"
+
 - name: Define path to drush for currently live build.
   set_fact:
     drush_live_bin: "{{ drush_live_bin | default('{{ live_symlink_dest }}/vendor/bin/drush') }}"
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index f57be10e..75523e18 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -16,6 +16,11 @@
   ansible.builtin.set_fact:
     webroot: "{{ webroot | default('web') }}"
 
+# This is passed from caller.
+- name: Gather last known good build number.
+  ansible.builtin.set_fact:
+    previous_build_number: "{{ previous_known_build_number }}"
+
 - name: Define build deploy path prefix.
   ansible.builtin.set_fact:
     deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_"
@@ -53,6 +58,10 @@
       ansible.builtin.set_fact:
         live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
 
+    - name: Set the previous deploy's path for later use where we need to manipulate the live site.
+      ansible.builtin.set_fact:
+        previous_deploy_path: "{{ previous_deploy_path | default('{{ deploy_path_prefix }}{{ previous_build_number }}') }}"
+
     - name: Overwrite deploy and live_symlink paths if SquashFS deploy.
       ansible.builtin.set_fact:
         deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
@@ -89,11 +98,6 @@
 #   register: last_build
 #   when: last_build_symlink.stat.exists
 
-# This is passed from caller.
-- name: Gather last known good build number.
-  ansible.builtin.set_fact:
-    previous_build_number: "{{ previous_known_build_number }}"
-
 # - set_fact:
 #     previous_build_number: "{{ last_build.stdout }}"
 #   when:
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 6b127f89..47a925f6 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -7,7 +7,20 @@
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:
-    - project_type == 'drupal8'
+    - project_type == "drupal8"
+
+# We want to also bring up the 'current' site with squashfs builds
+- name: Disable live site maintenance mode if this is a squashfs build.
+  ansible.builtin.command:
+    cmd: "{{ previous_drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ previous_deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  args:
+    chdir: "{{ previous_deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  when:
+    - project_type == "drupal8"
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
 
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Disable maintenance mode D7.
@@ -17,4 +30,16 @@
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
   when:
-    - project_type == 'drupal7'
+    - project_type == "drupal7"
+
+# We want to also bring up the 'current' site with squashfs builds
+- name: Disable maintenance mode D7 if this is a squashfs build.
+  ansible.builtin.shell:
+    cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0"
+    chdir: "{{ previous_deploy_path }}/{{ webroot }}/sites/{{ site.folder }}"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"
+  when:
+    - project_type == "drupal7"
+    - deploy_code.mount_type is defined
+    - deploy_code.mount_type == "squashfs"
diff --git a/roles/maintenance_mode/maintenance_mode-nginx/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-nginx/tasks/main.yml
index 65531755..5f5377da 100644
--- a/roles/maintenance_mode/maintenance_mode-nginx/tasks/main.yml
+++ b/roles/maintenance_mode/maintenance_mode-nginx/tasks/main.yml
@@ -1,8 +1,8 @@
 ---
-- include_tasks: "{{ maintenance_mode.operation }}.yml"
+- ansible.builtin.include_tasks: "{{ maintenance_mode.operation }}.yml"
 
 - name: Reload Nginx.
-  service:
+  ansible.builtin.service:
     name: nginx
     state: reloaded
   become: true
diff --git a/roles/maintenance_mode/maintenance_mode-nginx/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-nginx/tasks/offline.yml
index abcd2246..78fe1df5 100644
--- a/roles/maintenance_mode/maintenance_mode-nginx/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-nginx/tasks/offline.yml
@@ -1,6 +1,6 @@
 ---
 - name: Disable active vhost.
-  file:
+  ansible.builtin.file:
     path: "/etc/nginx/sites-enabled/{{ domain.server_name }}.conf"
     state: absent
   with_items: "{{ maintenance_mode_nginx.domains }}"
@@ -9,7 +9,7 @@
   become: true
 
 - name: Enable maintenance vhost.
-  file:
+  ansible.builtin.file:
     src: "/etc/nginx/sites-available/{{ domain.server_name }}-maintenance.conf"
     dest: "/etc/nginx/sites-enabled/{{ domain.server_name }}-maintenance.conf"
     state: link
diff --git a/roles/maintenance_mode/maintenance_mode-nginx/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-nginx/tasks/online.yml
index 59d59c5d..7ad26114 100644
--- a/roles/maintenance_mode/maintenance_mode-nginx/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-nginx/tasks/online.yml
@@ -1,6 +1,6 @@
 ---
 - name: Disable maintenance vhost.
-  file:
+  ansible.builtin.file:
     path: "/etc/nginx/sites-enabled/{{ domain.server_name }}-maintenance.conf"
     state: absent
   with_items: "{{ maintenance_mode_nginx.domains }}"
@@ -9,7 +9,7 @@
   become: true
 
 - name: Enable active vhost.
-  file:
+  ansible.builtin.file:
     src: "/etc/nginx/sites-available/{{ domain.server_name }}.conf"
     dest: "/etc/nginx/sites-enabled/{{ domain.server_name }}.conf"
     state: link
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
index 15b5cac8..ea8de053 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/main.yml
@@ -5,4 +5,4 @@
     loop_var: domain
   run_once: true
   when:
-    - maintenance_mode_statuscake.domains | length > 0
\ No newline at end of file
+    - maintenance_mode_statuscake.domains | length > 0
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
index 62081308..1aeb574f 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/offline.yml
@@ -1,7 +1,7 @@
 ---
 - name: Pause StatusCake check.
   ansible.builtin.uri:
-    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
+    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id | string }}"
     method: PUT
     body_format: form-urlencoded
     body:
diff --git a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
index 5818ed33..1bed8ec3 100644
--- a/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-statuscake/tasks/online.yml
@@ -1,7 +1,7 @@
 ---
 - name: Resume StatusCake check.
   ansible.builtin.uri:
-    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id }}"
+    url: "https://api.statuscake.com/v1/uptime/{{ domain.statuscake_check_id | string }}"
     method: PUT
     body_format: form-urlencoded
     body:
diff --git a/roles/maintenance_mode/tasks/main.yml b/roles/maintenance_mode/tasks/main.yml
index 2fbf77ee..ed0ef7f0 100644
--- a/roles/maintenance_mode/tasks/main.yml
+++ b/roles/maintenance_mode/tasks/main.yml
@@ -1,5 +1,5 @@
 ---
-- include_role:
+- ansible.builtin.include_role:
     name: "maintenance_mode/maintenance_mode-{{ maintenance_mode.mode }}"
   when:
     - deploy_operation != 'cleanup'

From f41783833c8037f0a03c0a3bf79e76895fa9e85a Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 26 Apr 2024 15:02:34 +0200
Subject: [PATCH 119/139] Adding extra healthcheck options for ECS services.
 (#463)

---
 docs/roles/deploy_container.md                 | 4 ++++
 roles/deploy_container/README.md               | 4 ++++
 roles/deploy_container/defaults/main.yml       | 4 ++++
 roles/deploy_container/tasks/action-create.yml | 7 +++++--
 4 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index 06847722..ed42996d 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -124,6 +124,10 @@ deploy_container:
       protocol: http
       path: /
       response_codes: "200"
+      # optional additional healthcheck settings
+      #interval: 60
+      #healthy_threshold_count: 3
+      #unhealthy_threshold_count: 5
     # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
     acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
       create_cert: false
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index 06847722..ed42996d 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -124,6 +124,10 @@ deploy_container:
       protocol: http
       path: /
       response_codes: "200"
+      # optional additional healthcheck settings
+      #interval: 60
+      #healthy_threshold_count: 3
+      #unhealthy_threshold_count: 5
     # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
     acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
       create_cert: false
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index ff99d2b9..98b23723 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -81,6 +81,10 @@ deploy_container:
       protocol: http
       path: /
       response_codes: "200"
+      # optional additional healthcheck settings
+      #interval: 60
+      #healthy_threshold_count: 3
+      #unhealthy_threshold_count: 5
     # Requires the deploy IAM user to have the managed AWSCertificateManagerFullAccess and AmazonRoute53FullAccess policies attached
     acm: # see https://github.com/codeenigma/ce-provision/tree/1.x/roles/aws/aws_acm
       create_cert: false
diff --git a/roles/deploy_container/tasks/action-create.yml b/roles/deploy_container/tasks/action-create.yml
index cc06163d..5dadd039 100644
--- a/roles/deploy_container/tasks/action-create.yml
+++ b/roles/deploy_container/tasks/action-create.yml
@@ -70,7 +70,7 @@
 
 - name: Fetch the aws_acm files.
   ansible.builtin.get_url:
-    url: "https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/{{ item }}/main.yml"
+    url: "https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/aws/aws_acm/{{ item }}/main.yml"
     dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/{{ item }}/main.yml"
   delegate_to: localhost
   with_items:
@@ -79,7 +79,7 @@
 
 - name: Fetch the aws_acm tasks.
   ansible.builtin.get_url:
-    url: https://raw.githubusercontent.com/codeenigma/ce-provision/1.x/roles/aws/aws_acm/tasks/main.yml
+    url: https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/aws/aws_acm/tasks/main.yml
     dest: "{{ _ce_deploy_base_dir }}/roles/aws_acm/tasks/main.yml"
   delegate_to: localhost
 
@@ -161,6 +161,9 @@
     vpc_id: "{{ _aws_ecs_cluster_vpc_id }}"
     health_check_protocol: "{{ deploy_container.aws_ecs.health_check.protocol }}"
     health_check_path: "{{ deploy_container.aws_ecs.health_check.path }}"
+    health_check_interval: "{{ deploy_container.aws_ecs.health_check.interval | default(omit) }}"
+    healthy_threshold_count: "{{ deploy_container.aws_ecs.health_check.healthy_threshold_count | default(omit) }}"
+    unhealthy_threshold_count: "{{ deploy_container.aws_ecs.health_check.unhealthy_threshold_count | default(omit) }}"
     successful_response_codes: "{{ deploy_container.aws_ecs.health_check.response_codes }}"
     target_type: ip
     targets: "{{ deploy_container.aws_ecs.targets }}"

From 8e01d772261e9311e7eed215174e785efb934020 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 26 Apr 2024 16:26:35 +0200
Subject: [PATCH 120/139] Adding behaviour to suspend autoscaling on deploy.
 (#465)

---
 docs/_Sidebar.md                              |  1 +
 docs/roles/_exit.md                           |  2 +
 docs/roles/_init.md                           | 41 ++++++++++++-------
 docs/roles/sync/database_sync.md              |  1 +
 .../sync/database_sync/database_sync-mysql.md |  7 ++++
 roles/_exit/README.md                         |  2 +
 roles/_exit/tasks/main.yml                    | 10 ++++-
 roles/_init/README.md                         | 41 ++++++++++++-------
 roles/_init/defaults/main.yml                 | 29 +++++++------
 roles/_init/tasks/main.yml                    | 10 +++++
 roles/sync/database_sync/README.md            |  1 +
 .../database_sync-mysql/README.md             |  7 ++++
 .../database_sync-mysql/tasks/sync.yml        |  1 +
 13 files changed, 110 insertions(+), 43 deletions(-)
 create mode 100644 docs/roles/_exit.md
 create mode 100644 roles/_exit/README.md

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index a58673b3..baa010a8 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -32,6 +32,7 @@
        - [MySQL backups](/roles/database_backup/database_backup-mysql)
      - [Deploy](/roles/deploy_code)
      - [Deploy container](/roles/deploy_container)
+     - [Exit](/roles/_exit)
      - [Init](/roles/_init)
      - [LHCI run](/roles/lhci_run)
      - [Maintenance Mode](/roles/maintenance_mode)
diff --git a/docs/roles/_exit.md b/docs/roles/_exit.md
new file mode 100644
index 00000000..6aa676a6
--- /dev/null
+++ b/docs/roles/_exit.md
@@ -0,0 +1,2 @@
+# Exit
+Mandatory role that must run after any other `ce-deploy` roles when executing a playbook.
diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 98179138..f31e8064 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -1,5 +1,15 @@
 # Init
-These variables **must** be set in the `deploy/common.yml` file, at least.
+Mandatory role that must run before any other `ce-edploy` roles when executing a playbook.
+
+These variables **must** be set in a common variables file if you do not wish to use defaults.
+
+In order to manipulate an AWS Autoscaling Group (ASG) your `deploy` user must have an AWS CLI profile for a user with the following IAM permissions:
+* `autoscaling:ResumeProcesses`
+* `autoscaling:SuspendProcesses`
+* `autoscaling:DescribeScalingProcessTypes`
+* `autoscaling:DescribeAutoScalingGroups`
+
+Set the `aws_asg.name` to the machine name of your ASG in order to automatically suspend and resume autoscaling on build.
 
 <!--TOC-->
 <!--ENDTOC-->
@@ -8,16 +18,24 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 ## Default variables
 ```yaml
 ---
-# Common defaults. Given the "_init" role is mandatory,
-# this will ensure defaults to other roles too.
-# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
-deploy_user: "deploy"
-# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
+# Common defaults - the "_init" role is mandatory so this will ensure defaults to other roles too.
+deploy_user: deploy # if you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
+# For MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here:
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+# @TODO only used by Drupal 7, can be removed with Drupal 7 deployments
+bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+# AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
+aws_asg:
+  name: "" # if the deploy is on an ASG put the name here
+  region: "eu-west-1"
+  suspend_processes: "Launch Terminate" # space separated string, see https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-suspend-resume-processes.html
+# Application specific variables.
 drupal:
   drush_verbose_output: false
-  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
-  truncate_cache_table: false
+  truncate_cache_table: false # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error
   sites:
     - folder: "default"
       public_files: "sites/default/files"
@@ -37,7 +55,7 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build - include all variables if used
+      # For syncing database and files on a feature branch initial build - include all variables if used:
       mysql_sync: {} # see sync/database_sync for docs
       #  mysqldump_params: "{{ _mysqldump_params }}"
       #  cleanup: true
@@ -50,11 +68,6 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-# Used for custom build time tools like cachetool
-bin_directory: "/home/{{ deploy_user }}/.bin"
-# Number of dumps/db to look up for cleanup.
-cleanup_history_depth: 50
-install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
 
 ```
 
diff --git a/docs/roles/sync/database_sync.md b/docs/roles/sync/database_sync.md
index 850b88ba..d4e8899a 100644
--- a/docs/roles/sync/database_sync.md
+++ b/docs/roles/sync/database_sync.md
@@ -1,5 +1,6 @@
 # Database sync
 Roles that sync databases between environments.
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index c0c2954b..ab07346d 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -1,5 +1,12 @@
 # Database sync - MySQL
 Sync MySQL databases between environments.
+
+In order to manipulate an AWS Autoscaling Group (ASG) your `deploy` user must have an AWS CLI profile for a user with the following IAM permissions:
+* `autoscaling:ResumeProcesses`
+* `autoscaling:SuspendProcesses`
+* `autoscaling:DescribeScalingProcessTypes`
+* `autoscaling:DescribeAutoScalingGroups`
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
diff --git a/roles/_exit/README.md b/roles/_exit/README.md
new file mode 100644
index 00000000..6aa676a6
--- /dev/null
+++ b/roles/_exit/README.md
@@ -0,0 +1,2 @@
+# Exit
+Mandatory role that must run after any other `ce-deploy` roles when executing a playbook.
diff --git a/roles/_exit/tasks/main.yml b/roles/_exit/tasks/main.yml
index 1a62b9af..0ba2da27 100644
--- a/roles/_exit/tasks/main.yml
+++ b/roles/_exit/tasks/main.yml
@@ -1,3 +1,9 @@
 ---
-
-# We do nothing there, just a placeholder for now.
\ No newline at end of file
+# If we are operating on an AWS ASG then resume autoscaling.
+- name: Enable all autoscale processes on ASG.
+  ansible.builtin.command: >
+    aws autoscaling resume-processes --auto-scaling-group-name {{ aws_asg.name }} --region {{ aws_asg.region }}
+  delegate_to: localhost
+  when:
+    - aws_asg.name is defined
+    - aws_asg.name | length > 0
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 98179138..f31e8064 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -1,5 +1,15 @@
 # Init
-These variables **must** be set in the `deploy/common.yml` file, at least.
+Mandatory role that must run before any other `ce-edploy` roles when executing a playbook.
+
+These variables **must** be set in a common variables file if you do not wish to use defaults.
+
+In order to manipulate an AWS Autoscaling Group (ASG) your `deploy` user must have an AWS CLI profile for a user with the following IAM permissions:
+* `autoscaling:ResumeProcesses`
+* `autoscaling:SuspendProcesses`
+* `autoscaling:DescribeScalingProcessTypes`
+* `autoscaling:DescribeAutoScalingGroups`
+
+Set the `aws_asg.name` to the machine name of your ASG in order to automatically suspend and resume autoscaling on build.
 
 <!--TOC-->
 <!--ENDTOC-->
@@ -8,16 +18,24 @@ These variables **must** be set in the `deploy/common.yml` file, at least.
 ## Default variables
 ```yaml
 ---
-# Common defaults. Given the "_init" role is mandatory,
-# this will ensure defaults to other roles too.
-# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
-deploy_user: "deploy"
-# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
+# Common defaults - the "_init" role is mandatory so this will ensure defaults to other roles too.
+deploy_user: deploy # if you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
+# For MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here:
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+# @TODO only used by Drupal 7, can be removed with Drupal 7 deployments
+bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+# AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
+aws_asg:
+  name: "" # if the deploy is on an ASG put the name here
+  region: "eu-west-1"
+  suspend_processes: "Launch Terminate" # space separated string, see https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-suspend-resume-processes.html
+# Application specific variables.
 drupal:
   drush_verbose_output: false
-  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
-  truncate_cache_table: false
+  truncate_cache_table: false # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error
   sites:
     - folder: "default"
       public_files: "sites/default/files"
@@ -37,7 +55,7 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build - include all variables if used
+      # For syncing database and files on a feature branch initial build - include all variables if used:
       mysql_sync: {} # see sync/database_sync for docs
       #  mysqldump_params: "{{ _mysqldump_params }}"
       #  cleanup: true
@@ -50,11 +68,6 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-# Used for custom build time tools like cachetool
-bin_directory: "/home/{{ deploy_user }}/.bin"
-# Number of dumps/db to look up for cleanup.
-cleanup_history_depth: 50
-install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
 
 ```
 
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 2e4eb0a8..7f06f695 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -1,14 +1,22 @@
 ---
-# Common defaults. Given the "_init" role is mandatory,
-# this will ensure defaults to other roles too.
-# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
-deploy_user: "deploy"
-# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here
+# Common defaults - the "_init" role is mandatory so this will ensure defaults to other roles too.
+deploy_user: deploy # if you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable
+# For MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here:
 _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table"
+# @TODO only used by Drupal 7, can be removed with Drupal 7 deployments
+bin_directory: "/home/{{ deploy_user }}/.bin"
+# Number of dumps/db to look up for cleanup.
+cleanup_history_depth: 50
+install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+# AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
+aws_asg:
+  name: "" # if the deploy is on an ASG put the name here
+  region: "eu-west-1"
+  suspend_processes: "Launch Terminate" # space separated string, see https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-suspend-resume-processes.html
+# Application specific variables.
 drupal:
   drush_verbose_output: false
-  # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error.
-  truncate_cache_table: false
+  truncate_cache_table: false # when set to true - truncate database table cache_container, a workaround to resolve the 'Cannot redeclare ...' error
   sites:
     - folder: "default"
       public_files: "sites/default/files"
@@ -28,7 +36,7 @@ drupal:
         - minute: "*/{{ 10 | random(start=1) }}"
           job: cron
       feature_branch: false # whether or not this build is a feature branch that should sync assets from another environment
-      # For syncing database and files on a feature branch initial build - include all variables if used
+      # For syncing database and files on a feature branch initial build - include all variables if used:
       mysql_sync: {} # see sync/database_sync for docs
       #  mysqldump_params: "{{ _mysqldump_params }}"
       #  cleanup: true
@@ -41,8 +49,3 @@ drupal:
 mautic:
   image_path: "media/images"
   force_install: false
-# Used for custom build time tools like cachetool
-bin_directory: "/home/{{ deploy_user }}/.bin"
-# Number of dumps/db to look up for cleanup.
-cleanup_history_depth: 50
-install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 75523e18..680ba19f 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -122,3 +122,13 @@
   ansible.builtin.include_tasks: "{{ project_type }}.yml"
   when:
     - _project_type_task_result.stat.exists
+
+# If we are operating on an AWS ASG then pause autoscaling.
+# @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html
+- name: Disable autoscale processes on ASG.
+  ansible.builtin.command: >
+    aws autoscaling suspend-processes --auto-scaling-group-name {{ aws_asg.name }} --scaling-processes {{ aws_asg.suspend_processes }} --region {{ aws_asg.region }}
+  delegate_to: localhost
+  when:
+    - aws_asg.name is defined
+    - aws_asg.name | length > 0
diff --git a/roles/sync/database_sync/README.md b/roles/sync/database_sync/README.md
index 850b88ba..d4e8899a 100644
--- a/roles/sync/database_sync/README.md
+++ b/roles/sync/database_sync/README.md
@@ -1,5 +1,6 @@
 # Database sync
 Roles that sync databases between environments.
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index c0c2954b..ab07346d 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -1,5 +1,12 @@
 # Database sync - MySQL
 Sync MySQL databases between environments.
+
+In order to manipulate an AWS Autoscaling Group (ASG) your `deploy` user must have an AWS CLI profile for a user with the following IAM permissions:
+* `autoscaling:ResumeProcesses`
+* `autoscaling:SuspendProcesses`
+* `autoscaling:DescribeScalingProcessTypes`
+* `autoscaling:DescribeAutoScalingGroups`
+
 <!--ROLEVARS-->
 ## Default variables
 ```yaml
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 3f94e47d..41266982 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -17,6 +17,7 @@
     - database.target.asg is defined
     - database.target.asg | length > 0
 
+# @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html
 - name: Disable ReplaceUnhealthy autoscale process on source ASG.
   ansible.builtin.command: >
     aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}

From 61471fd0ba4b0d352aaab0bf79da63e1e86851c2 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 26 Apr 2024 16:38:52 +0200
Subject: [PATCH 121/139] Ecs healthcheck vars pr 1.x (#467)

* Adding extra healthcheck options for ECS services.

* Renaming scale up/down to in/out to avoid confusion.

* Spotted annoying typo.
---
 docs/roles/deploy_container.md                 | 4 ++--
 roles/_init/README.md                          | 2 +-
 roles/deploy_container/README.md               | 4 ++--
 roles/deploy_container/defaults/main.yml       | 4 ++--
 roles/deploy_container/tasks/action-create.yml | 4 ++--
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/roles/deploy_container.md b/docs/roles/deploy_container.md
index ed42996d..15955a56 100644
--- a/docs/roles/deploy_container.md
+++ b/docs/roles/deploy_container.md
@@ -88,8 +88,8 @@ deploy_container:
       - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
-    service_autoscale_up_cooldown: 120
-    service_autoscale_down_cooldown: 120
+    service_autoscale_in_cooldown: 120 # scale down
+    service_autoscale_out_cooldown: 120 # scale up
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
diff --git a/roles/_init/README.md b/roles/_init/README.md
index f31e8064..8d4fa92e 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -1,5 +1,5 @@
 # Init
-Mandatory role that must run before any other `ce-edploy` roles when executing a playbook.
+Mandatory role that must run before any other `ce-deploy` roles when executing a playbook.
 
 These variables **must** be set in a common variables file if you do not wish to use defaults.
 
diff --git a/roles/deploy_container/README.md b/roles/deploy_container/README.md
index ed42996d..15955a56 100644
--- a/roles/deploy_container/README.md
+++ b/roles/deploy_container/README.md
@@ -88,8 +88,8 @@ deploy_container:
       - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
-    service_autoscale_up_cooldown: 120
-    service_autoscale_down_cooldown: 120
+    service_autoscale_in_cooldown: 120 # scale down
+    service_autoscale_out_cooldown: 120 # scale up
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
diff --git a/roles/deploy_container/defaults/main.yml b/roles/deploy_container/defaults/main.yml
index 98b23723..169fb359 100644
--- a/roles/deploy_container/defaults/main.yml
+++ b/roles/deploy_container/defaults/main.yml
@@ -45,8 +45,8 @@ deploy_container:
       - example-cluster-dev-b
     # See docs for values: https://docs.aws.amazon.com/autoscaling/application/APIReference/API_TargetTrackingScalingPolicyConfiguration.html
     service_autoscale_metric_type: ECSServiceAverageCPUUtilization
-    service_autoscale_up_cooldown: 120
-    service_autoscale_down_cooldown: 120
+    service_autoscale_in_cooldown: 120 # scale down
+    service_autoscale_out_cooldown: 120 # scale up
     service_autoscale_target_value: 70 # the value to trigger a scaling event at
     service_public_container_ip: false # set to true to make containers appear on an EIP - more details: https://stackoverflow.com/a/66802973
     service_enable_ssm: false # set to true to allow arbitrary command execution on containers via the AWS API
diff --git a/roles/deploy_container/tasks/action-create.yml b/roles/deploy_container/tasks/action-create.yml
index 5dadd039..58ebf27a 100644
--- a/roles/deploy_container/tasks/action-create.yml
+++ b/roles/deploy_container/tasks/action-create.yml
@@ -328,8 +328,8 @@
     target_tracking_scaling_policy_configuration:
       PredefinedMetricSpecification:
         PredefinedMetricType: "{{ deploy_container.aws_ecs.service_autoscale_metric_type }}"
-      ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_up_cooldown }}"
-      ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_down_cooldown }}"
+      ScaleInCooldown: "{{ deploy_container.aws_ecs.service_autoscale_in_cooldown }}"
+      ScaleOutCooldown: "{{ deploy_container.aws_ecs.service_autoscale_out_cooldown }}"
       DisableScaleIn: false
       TargetValue: "{{ deploy_container.aws_ecs.service_autoscale_target_value }}"
   delegate_to: localhost

From c9897f82e161b4f7b36f15ebd7d76fa0ab865836 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 20 May 2024 15:30:52 +0200
Subject: [PATCH 122/139] Fixing MySQL servers with unix socket.

---
 roles/database_backup/database_backup-mysql/defaults/main.yml   | 2 ++
 .../database_backup-mysql/tasks/deploy-rolling.yml              | 1 +
 2 files changed, 3 insertions(+)

diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml
index e7351d89..1401d1ca 100644
--- a/roles/database_backup/database_backup-mysql/defaults/main.yml
+++ b/roles/database_backup/database_backup-mysql/defaults/main.yml
@@ -5,6 +5,8 @@ mysql_backup:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
   mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
+  # Uncomment to login with MySQL socket instead of TCP/IP (e.g. for MariaDB after secure set-up)
+  #mysql_unix_socket: /run/mysqld/mysqld.sock
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 30fbaa62..65835f58 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -19,6 +19,7 @@
     state: present
     config_file: "{{ database.credentials_file }}"
     config_overrides_defaults: true
+    login_unix_socket: "{{ mysql_client.mysql_unix_socket | default(omit) }}"
   run_once: true
 
 - name: Check if the previous database exists.

From 70192ad67506b61522fcf709ca204b24eeb85a89 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 20 May 2024 15:32:18 +0200
Subject: [PATCH 123/139] Bad var name.

---
 .../database_backup-mysql/tasks/deploy-rolling.yml              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
index 65835f58..458e0978 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml
@@ -19,7 +19,7 @@
     state: present
     config_file: "{{ database.credentials_file }}"
     config_overrides_defaults: true
-    login_unix_socket: "{{ mysql_client.mysql_unix_socket | default(omit) }}"
+    login_unix_socket: "{{ mysql_backup.mysql_unix_socket | default(omit) }}"
   run_once: true
 
 - name: Check if the previous database exists.

From ce9b24a419f2cb293c89f4dfbcdaf6f83804dc56 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 20 May 2024 18:11:33 +0200
Subject: [PATCH 124/139] Extending MySQL Unix socket support.

---
 .../database_backup-mysql/tasks/cleanup-rolling.yml              | 1 +
 .../database_backup/database_backup-mysql/tasks/deploy-dump.yml  | 1 +
 .../database_backup/database_backup-mysql/tasks/deploy-none.yml  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
index 374e65ca..a94ccc2a 100644
--- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml
@@ -6,5 +6,6 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
+    login_unix_socket: "{{ mysql_backup.mysql_unix_socket | default(omit) }}"
   with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }}  end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }}
   run_once: true
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
index 49bda27e..28422dd5 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml
@@ -20,6 +20,7 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
+    login_unix_socket: "{{ mysql_backup.mysql_unix_socket | default(omit) }}"
   when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true
 
diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
index 4c4b1624..2a8ef973 100644
--- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
+++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml
@@ -20,5 +20,6 @@
     login_host: "{{ _mysql_host }}"
     login_user: "{{ _mysql_user }}"
     login_password: "{{ _mysql_password }}"
+    login_unix_socket: "{{ mysql_backup.mysql_unix_socket | default(omit) }}"
   when: previous_build_number == 0 or _build_db_status.rc == 1
   run_once: true

From e8e571f7f252849a0b43ac3cdeaba698d98a8141 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Thu, 23 May 2024 16:05:33 +0200
Subject: [PATCH 125/139] Bug fixes pr 1.x (#469)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.

* Adding support for specifying path to Ansible and path to Python interpreter.

* Moving installation of cachetool to application includes, where they are PHP apps.

* Fixing minor typo.
---
 docs/roles/_init.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index f31e8064..8d4fa92e 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -1,5 +1,5 @@
 # Init
-Mandatory role that must run before any other `ce-edploy` roles when executing a playbook.
+Mandatory role that must run before any other `ce-deploy` roles when executing a playbook.
 
 These variables **must** be set in a common variables file if you do not wish to use defaults.
 

From a236c59da803dd19fc49ac2caeb20481a04c9246 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 24 May 2024 12:49:24 +0200
Subject: [PATCH 126/139] Lock file pr 1.x (#471)

* Adding pre-flight checks and lock file behaviour to ce-deploy.

* Moving lock file paths to variables.
---
 docs/roles/_init.md                           |  3 ++
 .../database_backup/database_backup-mysql.md  |  2 ++
 roles/_exit/tasks/main.yml                    |  5 +++
 roles/_init/README.md                         |  3 ++
 roles/_init/defaults/main.yml                 |  3 ++
 roles/_init/tasks/main.yml                    | 31 +++++++++++++++++++
 .../database_backup-mysql/README.md           |  2 ++
 7 files changed, 49 insertions(+)

diff --git a/docs/roles/_init.md b/docs/roles/_init.md
index 8d4fa92e..a1e37f95 100644
--- a/docs/roles/_init.md
+++ b/docs/roles/_init.md
@@ -27,6 +27,9 @@ bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
 install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+ce_deploy_version: 1.x
+lock_file: /tmp/ce-deploy-lock
+provision_lock_file: /tmp/ce-provision-lock # must match _init.lock_file in ce-provision
 # AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
 aws_asg:
   name: "" # if the deploy is on an ASG put the name here
diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md
index 2766a4f3..f23c9c34 100644
--- a/docs/roles/database_backup/database_backup-mysql.md
+++ b/docs/roles/database_backup/database_backup-mysql.md
@@ -24,6 +24,8 @@ mysql_backup:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
   mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
+  # Uncomment to login with MySQL socket instead of TCP/IP (e.g. for MariaDB after secure set-up)
+  #mysql_unix_socket: /run/mysqld/mysqld.sock
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:
diff --git a/roles/_exit/tasks/main.yml b/roles/_exit/tasks/main.yml
index 0ba2da27..640f3a18 100644
--- a/roles/_exit/tasks/main.yml
+++ b/roles/_exit/tasks/main.yml
@@ -7,3 +7,8 @@
   when:
     - aws_asg.name is defined
     - aws_asg.name | length > 0
+
+- name: Delete the lock file.
+  ansible.builtin.file:
+    path: "{{ lock_file }}"
+    state: absent
diff --git a/roles/_init/README.md b/roles/_init/README.md
index 8d4fa92e..a1e37f95 100644
--- a/roles/_init/README.md
+++ b/roles/_init/README.md
@@ -27,6 +27,9 @@ bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
 install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+ce_deploy_version: 1.x
+lock_file: /tmp/ce-deploy-lock
+provision_lock_file: /tmp/ce-provision-lock # must match _init.lock_file in ce-provision
 # AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
 aws_asg:
   name: "" # if the deploy is on an ASG put the name here
diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml
index 7f06f695..00c14563 100644
--- a/roles/_init/defaults/main.yml
+++ b/roles/_init/defaults/main.yml
@@ -8,6 +8,9 @@ bin_directory: "/home/{{ deploy_user }}/.bin"
 # Number of dumps/db to look up for cleanup.
 cleanup_history_depth: 50
 install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app
+ce_deploy_version: 1.x
+lock_file: /tmp/ce-deploy-lock
+provision_lock_file: /tmp/ce-provision-lock # must match _init.lock_file in ce-provision
 # AWS ASG variables to allow for the suspension of autoscaling during a code deployment.
 aws_asg:
   name: "" # if the deploy is on an ASG put the name here
diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 680ba19f..1b53f4b2 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -1,4 +1,35 @@
 ---
+- name: Version check.
+  ansible.builtin.debug:
+    msg: "Using ce-deploy {{ ce_deploy_version }}"
+
+- name: Check for a ce-provision lock file.
+  ansible.builtin.stat:
+    path: "{{ provision_lock_file }}"
+  register: _ce_provision_lock
+
+- name: Abort if ce-provision lock file exists.
+  when: _ce_provision_lock.stat.exists is defined and _ce_provision_lock.stat.exists
+  block:
+    - name: Abort if ce-provision lock file is found.
+      ansible.builtin.debug:
+        msg: "ce-provision lock file discovered, an infrastructure build is in progress! If this is not the case, login to the affected server and delete the file at {{ provision_lock_file }}."
+    - ansible.builtin.meta: end_play
+
+- name: Check OS family.
+  when: ansible_os_family == "Windows"
+  block:
+    - name: Abort if target is a Windows server.
+      ansible.builtin.debug:
+        msg: "ce-deploy currently only supports Linux like operating systems, and works best with Debian Linux."
+    - ansible.builtin.meta: end_play
+
+- name: Set a lock file.
+  ansible.builtin.file:
+    path: "{{ lock_file }}"
+    state: touch
+    mode: 0644
+
 # Ensure default values for common variables.
 - name: Define deploy user.
   ansible.builtin.set_fact:
diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md
index 2766a4f3..f23c9c34 100644
--- a/roles/database_backup/database_backup-mysql/README.md
+++ b/roles/database_backup/database_backup-mysql/README.md
@@ -24,6 +24,8 @@ mysql_backup:
   mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here
   # Location on deploy server where the generated MySQL password will be stashed - should be temporary storage
   mysql_password_path: "/tmp/.ce-deploy/{{ project_name }}_{{ build_type }}_{{ build_number }}"
+  # Uncomment to login with MySQL socket instead of TCP/IP (e.g. for MariaDB after secure set-up)
+  #mysql_unix_socket: /run/mysqld/mysqld.sock
   # Number of dumps/db to keep. Note this is independant from the build codebases.
   keep: 10
   # This can be one of the following:

From 202e5be8584a9d5f4b2346452257d585b1964d7e Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 27 May 2024 18:01:33 +0200
Subject: [PATCH 127/139] Bug fixes pr 1.x (#476)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.

* Adding support for specifying path to Ansible and path to Python interpreter.

* Moving installation of cachetool to application includes, where they are PHP apps.

* Fixing minor typo.

* Cleaning up namespacing.

* Making quotes consistent.

* Making quotes consistent.

* Making quotes consistent.

* Allowing for Drupal cron rollback on revert.

* Only running maintenance mode enable on deploy.
---
 roles/_meta/deploy-drupal8/tasks/main.yml     | 32 +++++++++----------
 .../cache_clear-drupal8/tasks/main.yml        |  4 +--
 roles/config_generate/tasks/main.yml          |  6 ++--
 .../cron/cron_database_backup/tasks/main.yml  |  2 +-
 roles/cron/cron_drupal8/tasks/cron.yml        |  2 +-
 roles/cron/cron_drupal8/tasks/job.yml         |  6 ++++
 roles/database_apply/tasks/main.yml           |  2 +-
 .../tasks/offline.yml                         |  2 ++
 .../tasks/online.yml                          |  2 ++
 roles/sanitize/admin_creds/tasks/main.yml     |  2 +-
 10 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml
index 3e66875b..c8a9223b 100644
--- a/roles/_meta/deploy-drupal8/tasks/main.yml
+++ b/roles/_meta/deploy-drupal8/tasks/main.yml
@@ -3,41 +3,41 @@
 # - single site
 # - multisites with a single database
 
-- import_role:
+- ansible.builtin.import_role:
     name: _init
-- import_role:
+- ansible.builtin.import_role:
     name: deploy_code
-- import_role:
+- ansible.builtin.import_role:
     name: composer
-- import_role:
+- ansible.builtin.import_role:
     name: npm
-- import_role:
+- ansible.builtin.import_role:
     name: maintenance_mode
   vars:
     maintenance_mode:
       operation: offline
-- import_role:
+- ansible.builtin.import_role:
     name: database_backup
-- import_role:
+- ansible.builtin.import_role:
     name: config_generate
-- import_role:
+- ansible.builtin.import_role:
     name: cache_clear/cache_clear-opcache
-- import_role:
+- ansible.builtin.import_role:
     name: database_apply
-- import_role:
+- ansible.builtin.import_role:
     name: sanitize/admin_creds
-- import_role:
+- ansible.builtin.import_role:
     name: live_symlink
-- import_role:
+- ansible.builtin.import_role:
     name: cache_clear/cache_clear-drupal8
-- import_role:
+- ansible.builtin.import_role:
     name: maintenance_mode
   vars:
     maintenance_mode:
       operation: online
-- import_role:
+- ansible.builtin.import_role:
     name: cron/cron_database_backup
-- import_role:
+- ansible.builtin.import_role:
     name: cron/cron_drupal8
-- import_role:
+- ansible.builtin.import_role:
     name: _exit
diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
index a0f960da..063bfe8d 100644
--- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml
@@ -11,7 +11,7 @@
   when:
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
-    - deploy_operation == 'cleanup'
+    - deploy_operation == "cleanup"
 
 - name: Clear Drupal cache.
   ansible.builtin.command:
@@ -53,4 +53,4 @@
     - drupal.truncate_cache_table
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
-    - deploy_operation == 'cleanup'
+    - deploy_operation == "cleanup"
diff --git a/roles/config_generate/tasks/main.yml b/roles/config_generate/tasks/main.yml
index 2af08e87..adc35b06 100644
--- a/roles/config_generate/tasks/main.yml
+++ b/roles/config_generate/tasks/main.yml
@@ -2,7 +2,7 @@
 - name: Generate project specific configuration.
   ansible.builtin.include_role:
     name: "config_generate/config_generate-{{ project_type }}"
-  when: deploy_operation == 'deploy'
+  when: deploy_operation == "deploy"
 
 - name: Generate additional templates.
   ansible.builtin.template:
@@ -13,7 +13,7 @@
     loop_var: template
   when:
     - config_generate.templates | length
-    - deploy_operation == 'deploy'
+    - deploy_operation == "deploy"
 
 - name: Create additional symlinks.
   ansible.builtin.file:
@@ -25,4 +25,4 @@
     loop_var: link
   when:
     - config_generate.symlinks | length
-    - deploy_operation == 'deploy'
\ No newline at end of file
+    - deploy_operation == "deploy"
\ No newline at end of file
diff --git a/roles/cron/cron_database_backup/tasks/main.yml b/roles/cron/cron_database_backup/tasks/main.yml
index ab8bc0a1..4cd4676c 100644
--- a/roles/cron/cron_database_backup/tasks/main.yml
+++ b/roles/cron/cron_database_backup/tasks/main.yml
@@ -6,4 +6,4 @@
   with_items: "{{ database_backup.engines }}"
   loop_control:
     loop_var: engine
-  when: deploy_operation == 'deploy'
\ No newline at end of file
+  when: deploy_operation == "deploy"
\ No newline at end of file
diff --git a/roles/cron/cron_drupal8/tasks/cron.yml b/roles/cron/cron_drupal8/tasks/cron.yml
index 1ed40cc3..de153574 100644
--- a/roles/cron/cron_drupal8/tasks/cron.yml
+++ b/roles/cron/cron_drupal8/tasks/cron.yml
@@ -6,4 +6,4 @@
   loop_control:
     loop_var: entry
   when:
-    - deploy_operation == 'deploy'
+    - (deploy_operation == "deploy") or (deploy_operation == "revert")
diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml
index b89a0b05..7fa23c3e 100644
--- a/roles/cron/cron_drupal8/tasks/job.yml
+++ b/roles/cron/cron_drupal8/tasks/job.yml
@@ -2,6 +2,12 @@
 - name: Define cron job command.
   ansible.builtin.set_fact:
     _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
+  when: deploy_operation == "deploy"
+
+- name: Revert cron job command.
+  ansible.builtin.set_fact:
+    _cron_job_command: "cd {{ previous_deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
+  when: deploy_operation == "revert"
 
 - name: Define cron job command if deferred (ASG).
   ansible.builtin.set_fact:
diff --git a/roles/database_apply/tasks/main.yml b/roles/database_apply/tasks/main.yml
index 89138a98..ae1ad8ee 100644
--- a/roles/database_apply/tasks/main.yml
+++ b/roles/database_apply/tasks/main.yml
@@ -2,5 +2,5 @@
 - name: Apply database updates.
   include_role:
     name: "database_apply/database_apply-{{ project_type }}"
-  when: deploy_operation == 'deploy'
+  when: deploy_operation == "deploy"
   run_once: true
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
index 78314024..d70f87d0 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml
@@ -9,6 +9,7 @@
   when:
     - previous_build_number > 0
     - project_type == 'drupal8'
+    - deploy_operation == "deploy" # skip on cleanup and revert, no need
 
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Enable maintenance mode D7.
@@ -20,3 +21,4 @@
   when:
     - previous_build_number > 0
     - project_type == 'drupal7'
+    - deploy_operation == "deploy"
diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
index 47a925f6..02646ff6 100644
--- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
+++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml
@@ -21,6 +21,7 @@
     - project_type == "drupal8"
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
+    - deploy_operation == "deploy"
 
 # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead.
 - name: Disable maintenance mode D7.
@@ -43,3 +44,4 @@
     - project_type == "drupal7"
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
+    - deploy_operation == "deploy"
diff --git a/roles/sanitize/admin_creds/tasks/main.yml b/roles/sanitize/admin_creds/tasks/main.yml
index 880844ba..bc6f511d 100644
--- a/roles/sanitize/admin_creds/tasks/main.yml
+++ b/roles/sanitize/admin_creds/tasks/main.yml
@@ -2,5 +2,5 @@
 - name: Revert admin credentials.
   include_role:
     name: "sanitize/admin_creds/admin_creds-{{ project_type }}"
-  when: deploy_operation == 'deploy'
+  when: deploy_operation == "deploy"
   run_once: true

From 9d7dbf754d2126b71abd7ba5a527a1aaf0f5d6e6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 27 May 2024 18:22:03 +0200
Subject: [PATCH 128/139] Adding a new Drupal 10 meta role. (#478)

* Adding a new Drupal 10 meta role.

* Bad copy/paste in docs.
---
 docs/_Sidebar.md                           |  1 +
 docs/roles/_meta/deploy-drupal10.md        | 19 +++++++++++
 roles/_meta/deploy-drupal10/README.md      | 19 +++++++++++
 roles/_meta/deploy-drupal10/tasks/main.yml | 39 ++++++++++++++++++++++
 4 files changed, 78 insertions(+)
 create mode 100644 docs/roles/_meta/deploy-drupal10.md
 create mode 100644 roles/_meta/deploy-drupal10/README.md
 create mode 100644 roles/_meta/deploy-drupal10/tasks/main.yml

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index baa010a8..b8c4d52e 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -37,6 +37,7 @@
      - [LHCI run](/roles/lhci_run)
      - [Maintenance Mode](/roles/maintenance_mode)
      - ["Meta"](/roles/_meta)
+       - [Drupal 10](/roles/_meta/deploy-drupal10)
        - [Drupal 7](/roles/_meta/deploy-drupal7)
        - [Drupal 8](/roles/_meta/deploy-drupal8)
        - [Matomo](/roles/_meta/deploy-matomo)
diff --git a/docs/roles/_meta/deploy-drupal10.md b/docs/roles/_meta/deploy-drupal10.md
new file mode 100644
index 00000000..03cd9e40
--- /dev/null
+++ b/docs/roles/_meta/deploy-drupal10.md
@@ -0,0 +1,19 @@
+# Drupal 10
+Role for deploying single Drupal instances, or multisites with a single database, of Drupal version 10.0.0 or higher.
+
+You should always top and tail this role with the `_init` and `_exit` roles and include the files with your Ansible variable overrides in via `vars_files`, here is an example playbook:
+
+```yaml
+---
+- hosts: web1.example.com
+  vars_files:
+    - vars/common.yml
+    - vars/prod.yml
+  roles:
+    - _init
+    - _meta/deploy-drupal10
+    - _exit
+```
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/roles/_meta/deploy-drupal10/README.md b/roles/_meta/deploy-drupal10/README.md
new file mode 100644
index 00000000..03cd9e40
--- /dev/null
+++ b/roles/_meta/deploy-drupal10/README.md
@@ -0,0 +1,19 @@
+# Drupal 10
+Role for deploying single Drupal instances, or multisites with a single database, of Drupal version 10.0.0 or higher.
+
+You should always top and tail this role with the `_init` and `_exit` roles and include the files with your Ansible variable overrides in via `vars_files`, here is an example playbook:
+
+```yaml
+---
+- hosts: web1.example.com
+  vars_files:
+    - vars/common.yml
+    - vars/prod.yml
+  roles:
+    - _init
+    - _meta/deploy-drupal10
+    - _exit
+```
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/roles/_meta/deploy-drupal10/tasks/main.yml b/roles/_meta/deploy-drupal10/tasks/main.yml
new file mode 100644
index 00000000..6b0d86ab
--- /dev/null
+++ b/roles/_meta/deploy-drupal10/tasks/main.yml
@@ -0,0 +1,39 @@
+---
+# Default role for Drupal 10 and upwards. This is suitable for:
+# - single site
+# - multisites with a single database
+
+- ansible.builtin.import_role:
+    name: deploy_code
+- ansible.builtin.import_role:
+    name: composer
+- ansible.builtin.import_role:
+    name: npm
+- ansible.builtin.import_role:
+    name: maintenance_mode
+  vars:
+    maintenance_mode:
+      operation: offline
+- ansible.builtin.import_role:
+    name: database_backup
+- ansible.builtin.import_role:
+    name: config_generate
+- ansible.builtin.import_role:
+    name: cache_clear/cache_clear-opcache
+- ansible.builtin.import_role:
+    name: database_apply
+- ansible.builtin.import_role:
+    name: sanitize/admin_creds
+- ansible.builtin.import_role:
+    name: live_symlink
+- ansible.builtin.import_role:
+    name: cache_clear/cache_clear-drupal8
+- ansible.builtin.import_role:
+    name: maintenance_mode
+  vars:
+    maintenance_mode:
+      operation: online
+- ansible.builtin.import_role:
+    name: cron/cron_database_backup
+- ansible.builtin.import_role:
+    name: cron/cron_drupal8

From 1e0cbe0a578cf715e19a102bcf616294cb6b8202 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Fri, 31 May 2024 17:58:57 +0300
Subject: [PATCH 129/139] r69347-db-import-from-deploy-server (#480)

* r69347-db-import-from-deploy-server

* r69347-db-import-from-deploy-server

* r69347-db-import-from-deploy-server

* typo fixes and tidying up
---
 .../database_sync-mysql/defaults/main.yml     |   6 +-
 .../database_sync-mysql/tasks/sync.yml        | 240 ++++++++++--------
 2 files changed, 137 insertions(+), 109 deletions(-)

diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
index 98ac19bf..359f9e55 100644
--- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
+++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml
@@ -13,8 +13,9 @@ mysql_sync:
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
         # This can be of types:
         # - rolling: (database backups). In that case we'll need build parameters.@todo
-        # - fixed: "fixed" database name
+        # - fixed: "fixed" database name # currently this var means the same as "dump"
         # - dump: Use an existing dump. In that case, the "database" variable is the absolute file path.
+        # This parameter is ignored if var 'path_on_deploy_server' is defined.
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
@@ -25,6 +26,9 @@ mysql_sync:
         dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
         # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
         asg: ""
+        # Uncomment and specify full path if you want to use your own dump file on the deploy server (gzip compressed).
+        # In this case it's the only variable you need in 'database.source'.
+        #path_on_deploy_server: "/home/{{ deploy_user }}/compressed_database_dump.gz"
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 41266982..0ae03be9 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -1,42 +1,38 @@
 ---
-- name: Get database source host region.
-  amazon.aws.ec2_metadata_facts:
-  register: mysql_sync_source_database_host_info
-  delegate_to: "{{ database.source.host }}"
+- name: Prepare source ASG.
+  block:
+  - name: Get database source host region.
+    amazon.aws.ec2_metadata_facts:
+    register: mysql_sync_source_database_host_info
+    delegate_to: "{{ database.source.host }}"
+
+  # @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html
+  - name: Disable ReplaceUnhealthy autoscale process on source ASG.
+    ansible.builtin.command: >
+      aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+    delegate_to: localhost
   when:
     - database.source.asg is defined
     - database.source.asg | length > 0
     - database.source.fresh_db is defined
     - database.source.fresh_db
-
-# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host.
-- name: Get database target host region.
-  amazon.aws.ec2_metadata_facts:
-  register: mysql_sync_target_database_host_info
+    - database.source.path_on_deploy_server is not defined
+
+- name: Prepare target ASG.
+  block:
+  # This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host.
+  - name: Get database target host region.
+    amazon.aws.ec2_metadata_facts:
+    register: mysql_sync_target_database_host_info
+
+  - name: Disable ReplaceUnhealthy autoscale process on target ASG.
+    ansible.builtin.command: >
+      aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+    delegate_to: localhost
   when:
     - database.target.asg is defined
     - database.target.asg | length > 0
 
-# @TODO - the autoscaling_group module can do this - https://docs.ansible.com/ansible/latest/collections/amazon/aws/autoscaling_group_module.html
-- name: Disable ReplaceUnhealthy autoscale process on source ASG.
-  ansible.builtin.command: >
-    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
-  delegate_to: localhost
-  when:
-    - database.source.asg is defined
-    - database.source.asg | length > 0
-    - database.source.fresh_db is defined
-    - database.source.fresh_db
-
-- name: Disable ReplaceUnhealthy autoscale process on target ASG.
-  ansible.builtin.command: >
-    aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
-  delegate_to: localhost
-  when:
-    - database.target.asg is defined
-    - database.target.asg | length > 0
-
-
 - name: Register bzip2 archive type vars.
   ansible.builtin.set_fact:
     archive_file_type: "bz2"
@@ -54,56 +50,75 @@
 - name: Register remote dump name (from database).
   ansible.builtin.set_fact:
     mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.{{ archive_file_type }}"
-
-- name: Get source last known good build number.
-  ansible.builtin.command:
-    argv:
-      - "/bin/sh"
-      - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh"
-      - "--build-id"
-      - "{{ database.source.build_id }}"
-  register: mysql_sync_source_build_number
-  delegate_to: localhost
-  when: database.source.type == 'rolling'
-
-- name: Register source database name.
-  ansible.builtin.set_fact:
-    mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}"
-  when: database.source.type == 'rolling'
+  when:
+    - database.source.path_on_deploy_server is not defined
+
+- name: Detect rolling DB name on source.
+  block:
+  - name: Get source last known good build number.
+    ansible.builtin.command:
+      argv:
+        - "/bin/sh"
+        - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh"
+        - "--build-id"
+        - "{{ database.source.build_id }}"
+    register: mysql_sync_source_build_number
+    delegate_to: localhost
+
+  - name: Register source database name.
+    ansible.builtin.set_fact:
+      mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}"
+  when:
+    - database.source.type is defined
+    - database.source.type == 'rolling'
+    - database.source.path_on_deploy_server is not defined
 
 - name: Register source database name.
   ansible.builtin.set_fact:
     mysql_sync_source_database: "{{ database.source.database }}"
-  when: not database.source.type == 'rolling'
-
-- name: Take a dump from source database.
-  ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}"
-  args:
-    executable: /bin/bash
-  delegate_to: "{{ database.source.host }}"
   when:
-    - database.source.fresh_db
-
-- name: Find source database host.
-  ansible.builtin.command:
-    cmd: "grep 'host' {{ database.source.credentials_file }}"
-  register: mysql_host_info_grep
+    - database.source.type is defined
+    - not database.source.type == 'rolling'
+    - database.source.path_on_deploy_server is not defined
+
+- name: Fetch the source database.
+  block:
+  - name: Take a dump from source database.
+    ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | {{ archival_command }} > {{ mysql_sync_source_dump_path }}"
+    args:
+      executable: /bin/bash
+    when:
+      - database.source.fresh_db
+
+  - name: Find source database host.
+    ansible.builtin.command:
+      cmd: "grep 'host' {{ database.source.credentials_file }}"
+    register: mysql_host_info_grep
+    when:
+      - not database.source.fresh_db
+
+  - name: Register source database host.
+    set_fact:
+      mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}"
+    when:
+      - not database.source.fresh_db
+
+  - name: Copy a nightly backup for the source database.
+    ansible.builtin.copy:
+      src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}"
+      dest: "{{ mysql_sync_source_dump_path }}"
+      remote_src: true
+    when:
+      - not database.source.fresh_db
+
+  - name: Fetch dump file.
+    ansible.builtin.fetch:
+      src: "{{ mysql_sync_source_dump_path }}"
+      dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}"
+      flat: true
   delegate_to: "{{ database.source.host }}"
-  when: not database.source.fresh_db
-
-- name: Register source database host.
-  set_fact:
-    mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}"
-  delegate_to: "{{ database.source.host }}"
-  when: not database.source.fresh_db
-
-- name: Copy a nightly backup for the source database.
-  ansible.builtin.copy:
-    src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}"
-    dest: "{{ mysql_sync_source_dump_path }}"
-    remote_src: true
-  delegate_to: "{{ database.source.host }}"
-  when: not database.source.fresh_db
+  when:
+    - database.source.path_on_deploy_server is not defined
 
 - name: Register tmp target dump name.
   ansible.builtin.set_fact:
@@ -122,29 +137,34 @@
       - "{{ database.target.build_id }}"
   register: mysql_sync_target_build_number
   delegate_to: localhost
-  when: database.target.type == 'rolling'
+  when:
+    - database.target.type == 'rolling'
 
 - name: Register target rolling database name.
   ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}"
-  when: database.target.type == 'rolling'
+  when:
+    - database.target.type == 'rolling'
 
 - name: Register target static database name.
   ansible.builtin.set_fact:
     mysql_sync_target_database: "{{ database.target.database }}"
-  when: not database.target.type == 'rolling'
-
-- name: Fetch dump file.
-  ansible.builtin.fetch:
-    src: "{{ mysql_sync_source_dump_path }}"
-    dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}"
-    flat: true
-  delegate_to: "{{ database.source.host }}"
+  when:
+    - not database.target.type == 'rolling'
 
 - name: Copy dump file to destination.
   ansible.builtin.copy:
     src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.{{ archive_file_type }}"
     dest: "{{ mysql_sync_target_dump_path }}"
+  when:
+    - database.source.path_on_deploy_server is not defined
+
+- name: Copy dump file from deploy server to destination.
+  ansible.builtin.copy:
+    src: "{{ database.source.path_on_deploy_server }}"
+    dest: "{{ mysql_sync_target_dump_path }}"
+  when:
+    - database.source.path_on_deploy_server is defined
 
 - name: Unpack dump file.
   ansible.builtin.shell: "{{ archival_command }} -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}"
@@ -174,32 +194,36 @@
     path: "{{ mysql_sync_target_dump_unpacked_path }}"
     state: absent
 
-- name: Delete temporary dump file on source.
-  ansible.builtin.file:
-    path: "{{ mysql_sync_source_dump_path }}"
-    state: absent
-  delegate_to: "{{ database.source.host }}"
-
-- name: Delete temporary dump file on deploy server.
-  ansible.builtin.file:
-    path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}"
-    state: absent
-  delegate_to: localhost
-  when:
-    - mysql_sync.cleanup
-  with_items:
-    - ".bz2"
-    - ".gz"
-
-- name: Enable all autoscale processes on source ASG.
-  ansible.builtin.command: >
-    aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
-  delegate_to: localhost
+- name: Clean up source.
+  block:
+  - name: Delete temporary dump file on source.
+    ansible.builtin.file:
+      path: "{{ mysql_sync_source_dump_path }}"
+      state: absent
+    delegate_to: "{{ database.source.host }}"
+
+  - name: Delete temporary dump file on deploy server.
+    ansible.builtin.file:
+      path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql{{ item }}"
+      state: absent
+    delegate_to: localhost
+    when:
+      - mysql_sync.cleanup
+    with_items:
+      - ".bz2"
+      - ".gz"
+
+  - name: Enable all autoscale processes on source ASG.
+    ansible.builtin.command: >
+      aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }}
+    delegate_to: localhost
+    when:
+      - database.source.asg is defined
+      - database.source.asg | length > 0
+      - database.source.fresh_db is defined
+      - database.source.fresh_db
   when:
-    - database.source.asg is defined
-    - database.source.asg | length > 0
-    - database.source.fresh_db is defined
-    - database.source.fresh_db
+    - database.source.path_on_deploy_server is not defined
 
 - name: Enable all autoscale processes on target ASG.
   ansible.builtin.command: >

From 207be4a6a8d1a95764b1c66ca39706e53887164d Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Wed, 5 Jun 2024 10:27:27 +0200
Subject: [PATCH 130/139] Bug fixes pr 1.x (#486)

* Fixing linting so CI can work.

* More linting fixes.

* Downgrading Ubuntu due to Docker issues with ce-dev and latest.

* Making pipefail shell commands use /bin/bash.

* Adding pipefail code to cachetool installer block.

* Adding executable for drush install checker.

* Fixing D7 ctools behaviour to match features handling.

* Jinja2 template looking for settings.php using wrong path.

* Fixing config_generate for Drupal.

* Making host checking more robust.

* Removing variable declaration that will break host check.

* Allow us to use deploy_code with a completely custom build and do nothing.

* Make PHP cachetool install optional.

* Fixing bug introduced by failed_when - need to check for return code on mount points.

* Moving where opcache clears get called so they can be excluded if necessary.

* Fixing linting errors around white space.

* The unarchive module requires 'dest' to be a directory.

* Fixing Drupal example templates.

* Logically splitting tests and removing LGD because it's broken.

* Adding extra plugin LGD needs.

* Clearing stat and apcu cache for PHP by default.

* Adding some perms handling for Drupal 8+ config sync where deploy and web user are different.

* Rolling back APCu cache by default as the package must be installed.

* Making the remount squashfs code more defensive by using a block.

* Trying with 'throttle' keyword to restrict to one machine at a time.

* Sadly 'throttle' seems to work on a task level only, not a block level.

* Adding hash salt generation for Drupal 8+.

* Trying a different syntax for passing params to the password plugin.

* Forgot the constants import in plugin.

* Reinstating the SOPS plugin - we *do* need it.

* Making online/offline tasks use current live code/database.

* We need build paths to bring the site back online if it's a squashfs build.

* Ensuring _common.sh includes /etc/profile.d for Python venv path.

* Making linter happy.

* Using the drush_live_bin var in sync jobs for Drupal 8+.

* Fixing database dump revert.

* Fixing gunzip command.

* Having to do a double step to unpack SQL file in /tmp.

* Load bash profiles before executing a build.

* Fixing linting error, use -n instead of ! -z.

* Adding support for specifying path to Ansible and path to Python interpreter.

* Moving installation of cachetool to application includes, where they are PHP apps.

* Fixing minor typo.

* Cleaning up namespacing.

* Making quotes consistent.

* Making quotes consistent.

* Making quotes consistent.

* Allowing for Drupal cron rollback on revert.

* Only running maintenance mode enable on deploy.

* Moving previous_deploy_path out of the squashfs block.

* Committed installer by accident.
---
 roles/_init/tasks/main.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml
index 1b53f4b2..6f0cd830 100644
--- a/roles/_init/tasks/main.yml
+++ b/roles/_init/tasks/main.yml
@@ -89,10 +89,6 @@
       ansible.builtin.set_fact:
         live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}"
 
-    - name: Set the previous deploy's path for later use where we need to manipulate the live site.
-      ansible.builtin.set_fact:
-        previous_deploy_path: "{{ previous_deploy_path | default('{{ deploy_path_prefix }}{{ previous_build_number }}') }}"
-
     - name: Overwrite deploy and live_symlink paths if SquashFS deploy.
       ansible.builtin.set_fact:
         deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}"
@@ -111,6 +107,10 @@
     - deploy_code.mount_type is defined
     - deploy_code.mount_type == "squashfs"
 
+- name: Set the previous deploy's path for later use where we need to manipulate the live site.
+  ansible.builtin.set_fact:
+    previous_deploy_path: "{{ previous_deploy_path | default('{{ deploy_path_prefix }}{{ previous_build_number }}') }}"
+
 # Gather last known good build directly from symlink.
 # This can happen:
 # - when the first builds failed,

From 2ea17bcd54ba24d40c92d4d80d57c7eae22020ae Mon Sep 17 00:00:00 2001
From: Chris Maiden <chrispmaiden@gmail.com>
Date: Thu, 6 Jun 2024 11:06:06 +0100
Subject: [PATCH 131/139] 472 Cache clearing for Symfony based projects (#473)

* 472 Ensure Mautic cache is cleared on every host

* 472 Add Symfony cache clear task

* 472 Make console path configurable

* 472 Fix command variable
---
 roles/cache_clear/cache_clear-mautic/tasks/main.yml    |  2 --
 roles/cache_clear/cache_clear-symfony/README.md        |  7 +++++++
 .../cache_clear/cache_clear-symfony/defaults/main.yml  |  2 ++
 roles/cache_clear/cache_clear-symfony/tasks/main.yml   | 10 ++++++++++
 4 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 roles/cache_clear/cache_clear-symfony/README.md
 create mode 100644 roles/cache_clear/cache_clear-symfony/defaults/main.yml
 create mode 100644 roles/cache_clear/cache_clear-symfony/tasks/main.yml

diff --git a/roles/cache_clear/cache_clear-mautic/tasks/main.yml b/roles/cache_clear/cache_clear-mautic/tasks/main.yml
index 9ecec98c..56884d16 100644
--- a/roles/cache_clear/cache_clear-mautic/tasks/main.yml
+++ b/roles/cache_clear/cache_clear-mautic/tasks/main.yml
@@ -2,11 +2,9 @@
 - name: set console path
   set_fact:
     mautic_console_bin: "env php {{ deploy_path }}/{{ webroot }}/app/console"
-  run_once: true
 
 - name: Clear cache.
   command:
     cmd: "{{ mautic_console_bin }} cache:clear --no-interaction --env=prod --no-warmup"
   become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
   become_user: "{{ www_user }}"
-  run_once: true
diff --git a/roles/cache_clear/cache_clear-symfony/README.md b/roles/cache_clear/cache_clear-symfony/README.md
new file mode 100644
index 00000000..8bb3fde2
--- /dev/null
+++ b/roles/cache_clear/cache_clear-symfony/README.md
@@ -0,0 +1,7 @@
+# Symfony
+Clears the Symfony application cache.
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+<!--ENDROLEVARS-->
diff --git a/roles/cache_clear/cache_clear-symfony/defaults/main.yml b/roles/cache_clear/cache_clear-symfony/defaults/main.yml
new file mode 100644
index 00000000..c90f2d13
--- /dev/null
+++ b/roles/cache_clear/cache_clear-symfony/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+cache_clear_symfony_env: prod
diff --git a/roles/cache_clear/cache_clear-symfony/tasks/main.yml b/roles/cache_clear/cache_clear-symfony/tasks/main.yml
new file mode 100644
index 00000000..689af2c2
--- /dev/null
+++ b/roles/cache_clear/cache_clear-symfony/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+- name: Set console path.
+  ansible.builtin.set_fact:
+    symfony_console_bin: "{{ symfony_console_bin | default('env php ' + deploy_path + '/bin/console') }}"
+
+- name: Clear cache.
+  ansible.builtin.command:
+    cmd: "{{ symfony_console_bin }} cache:clear --no-interaction --env={{ cache_clear_symfony_env }} --no-warmup"
+  become: "{{ 'no' if www_user == deploy_user else 'yes' }}"
+  become_user: "{{ www_user }}"

From 2ed8f458202b2cbd645bfc544ede736e86056677 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Fri, 19 Jul 2024 19:32:25 +0200
Subject: [PATCH 132/139] Feature branching pr 1.x (#488)

* Adding syncing for feature branches.

* Cannot loop over import_role, changing for include_role.

* Adding ability to specify an exact filename for a settings template for Drupal 8 and above.

* First pass at NGINX vhost handling in ce-deploy.

* Adding in the SSL role from ce-provision.

* Fixing role paths and ensuring NGINX ssl.yml is available.

* Fixing NGINX role location.

* Tweaking location of domain.yml so it loads the templates correctly.
---
 roles/deploy_code/defaults/main.yml |   8 +++
 roles/deploy_code/tasks/deploy.yml  | 107 ++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+)

diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index 7b68c3a7..b8220475 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -1,5 +1,13 @@
 ---
+# Required for NGINX config in feature branching.
+_ce_provision_data_dir: /home/deploy/ce-deploy/data
+
 deploy_code:
+  # Feature branching config.
+  feature_branch:
+    enabled: false
+    # NGINX domains to build, see ce-provision for docs: https://github.com/codeenigma/ce-provision/blob/2.x/roles/debian/nginx/defaults/main.yml#L133
+    domains: []
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml
index 0d15425c..032e1577 100644
--- a/roles/deploy_code/tasks/deploy.yml
+++ b/roles/deploy_code/tasks/deploy.yml
@@ -38,3 +38,110 @@
     loop_var: link
   when:
     - deploy_code.symlinks | length
+
+# Additional vhost handling for feature branch builds.
+
+# Fetch the NGINX domain handling tasks from ce-provision.
+- name: Fetch the nginx role files from ce-provision.
+  when: deploy_code.feature_branch.enabled
+  delegate_to: localhost
+  block:
+    - name: Ensure the nginx directory exists.
+      ansible.builtin.file:
+        path: "{{ _ce_deploy_base_dir }}/roles/debian/nginx/{{ item }}"
+        state: directory
+        mode: '0755'
+      delegate_to: localhost
+      with_items:
+        - tasks
+        - defaults
+        - templates
+
+    - name: Ensure the ssl directory exists.
+      ansible.builtin.file:
+        path: "{{ _ce_deploy_base_dir }}/roles/debian/ssl/{{ item }}"
+        state: directory
+        mode: '0755'
+      delegate_to: localhost
+      with_items:
+        - tasks
+        - defaults
+        - templates
+
+    - name: Fetch nginx domain.yml.
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/nginx/tasks/domain.yml
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/nginx/domain.yml"
+
+    # Because of the way it is called, this needs putting into the playbook directory.
+    - name: Fetch nginx ssl.yml.
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/nginx/tasks/ssl.yml
+        dest: "{{ _ce_deploy_build_dir }}/deploy/ssl.yml"
+
+    - name: Fetch nginx defaults.
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/nginx/defaults/main.yml
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/nginx/defaults/main.yml"
+
+    - name: Fetch nginx templates.
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/nginx/templates/{{ item }}"
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/nginx/templates/{{ item }}"
+      with_items:
+        - vhosts.j2
+        - cloudwatch-vhost.json.j2
+
+    - name: Fetch ssl tasks.
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/ssl/tasks/{{ item }}.yml"
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/ssl/tasks/{{ item }}.yml"
+      with_items:
+        - copy
+        - generate
+        - letsencrypt
+        - main
+        - manual
+        - selfsigned
+        - unmanaged
+
+    - name: Fetch ssl defaults.
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/ssl/defaults/main.yml
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/ssl/defaults/main.yml"
+
+    - name: Fetch ssl templates.
+      ansible.builtin.get_url:
+        url: "https://raw.githubusercontent.com/codeenigma/ce-provision/2.x/roles/debian/ssl/templates/{{ item }}"
+        dest: "{{ _ce_deploy_base_dir }}/roles/debian/ssl/templates/{{ item }}"
+      with_items:
+        - le_cron.sh.j2
+
+# Generate the NGINX vhost.
+- name: Create vhost.
+  when:
+    - deploy_code.feature_branch.domains is defined
+    - deploy_code.feature_branch.domains | length > 0
+    - deploy_code.feature_branch.enabled
+  become: true
+  block:
+    - name: Generate domain specific configuration.
+      ansible.builtin.include_tasks: "{{ _ce_deploy_base_dir }}/roles/debian/nginx/domain.yml"
+      with_items: "{{ deploy_code.feature_branch.domains }}"
+      loop_control:
+        loop_var: domain
+
+    - name: Test NGINX configuration.
+      ansible.builtin.command: nginx -t
+      register: _nginx_test_result
+      failed_when: false
+
+    - name: Display NGINX test result.
+      ansible.builtin.debug:
+        msg: "{{ _nginx_test_result.stderr }}"
+
+    - name: Ensure NGINX is reloaded.
+      ansible.builtin.service:
+        name: nginx
+        state: reloaded
+      when: _nginx_test_result.rc == 0

From 04f8c1cbc277a5c8306d57075b041ff00eee4d56 Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 6 Aug 2024 11:01:47 +0300
Subject: [PATCH 133/139] additional_symlink_task_fix_for_squashfs (#494)

* additional_symlink_task_fix_for_squashfs

* additional_symlink_task_fix_for_squashfs

* additional_symlink_task_fix_for_squashfs

* additional_symlink_task_fixes

* additional_symlink_task_fixes
---
 docs/roles/composer.md                  | 4 +++-
 docs/roles/config_generate.md           | 4 +++-
 docs/roles/deploy_code.md               | 4 +++-
 docs/roles/npm.md                       | 4 +++-
 roles/composer/README.md                | 4 +++-
 roles/composer/defaults/main.yml        | 4 +++-
 roles/composer/tasks/main.yml           | 1 +
 roles/config_generate/README.md         | 4 +++-
 roles/config_generate/defaults/main.yml | 4 +++-
 roles/config_generate/tasks/main.yml    | 1 +
 roles/deploy_code/README.md             | 4 +++-
 roles/deploy_code/defaults/main.yml     | 4 +++-
 roles/deploy_code/tasks/deploy.yml      | 3 ++-
 roles/live_symlink/defaults/main.yml    | 4 +++-
 roles/live_symlink/tasks/main.yml       | 2 +-
 roles/npm/README.md                     | 4 +++-
 roles/npm/defaults/main.yml             | 4 +++-
 roles/npm/tasks/main.yml                | 1 +
 18 files changed, 45 insertions(+), 15 deletions(-)

diff --git a/docs/roles/composer.md b/docs/roles/composer.md
index 74765170..47e40b29 100644
--- a/docs/roles/composer.md
+++ b/docs/roles/composer.md
@@ -14,11 +14,13 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/docs/roles/config_generate.md b/docs/roles/config_generate.md
index fe7aecfe..b6b46aba 100644
--- a/docs/roles/config_generate.md
+++ b/docs/roles/config_generate.md
@@ -11,11 +11,13 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index 5d4cc722..634dbbe5 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -89,11 +89,13 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/docs/roles/npm.md b/docs/roles/npm.md
index 97f06cdc..5a17e577 100644
--- a/docs/roles/npm.md
+++ b/docs/roles/npm.md
@@ -18,11 +18,13 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/composer/README.md b/roles/composer/README.md
index 74765170..47e40b29 100644
--- a/roles/composer/README.md
+++ b/roles/composer/README.md
@@ -14,11 +14,13 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/composer/defaults/main.yml b/roles/composer/defaults/main.yml
index d0dfbd1d..314f8bd0 100644
--- a/roles/composer/defaults/main.yml
+++ b/roles/composer/defaults/main.yml
@@ -7,11 +7,13 @@ composer:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/composer/tasks/main.yml b/roles/composer/tasks/main.yml
index bba5e02a..5bf70693 100644
--- a/roles/composer/tasks/main.yml
+++ b/roles/composer/tasks/main.yml
@@ -25,6 +25,7 @@
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
+    force: "{{ link.force | default(false) }}"
   with_items: "{{ composer.symlinks }}"
   loop_control:
     loop_var: link
diff --git a/roles/config_generate/README.md b/roles/config_generate/README.md
index fe7aecfe..b6b46aba 100644
--- a/roles/config_generate/README.md
+++ b/roles/config_generate/README.md
@@ -11,11 +11,13 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/config_generate/defaults/main.yml b/roles/config_generate/defaults/main.yml
index 8f1987eb..5b4f50cf 100644
--- a/roles/config_generate/defaults/main.yml
+++ b/roles/config_generate/defaults/main.yml
@@ -3,11 +3,13 @@ config_generate:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/config_generate/tasks/main.yml b/roles/config_generate/tasks/main.yml
index adc35b06..dacf9615 100644
--- a/roles/config_generate/tasks/main.yml
+++ b/roles/config_generate/tasks/main.yml
@@ -20,6 +20,7 @@
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
+    force: "{{ link.force | default(false) }}"
   with_items: "{{ config_generate.symlinks }}"
   loop_control:
     loop_var: link
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index 5d4cc722..634dbbe5 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -89,11 +89,13 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml
index b8220475..62e50b00 100644
--- a/roles/deploy_code/defaults/main.yml
+++ b/roles/deploy_code/defaults/main.yml
@@ -11,11 +11,13 @@ deploy_code:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml
index 032e1577..a9d74d2e 100644
--- a/roles/deploy_code/tasks/deploy.yml
+++ b/roles/deploy_code/tasks/deploy.yml
@@ -28,11 +28,12 @@
     - deploy_code.templates | length
     - deploy_operation == 'deploy'
 
-- name: Create additional symlinks.
+- name: Create additional symlinks in build dir.
   ansible.builtin.file:
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
+    force: "{{ link.force | default(false) }}"
   with_items: "{{ deploy_code.symlinks }}"
   loop_control:
     loop_var: link
diff --git a/roles/live_symlink/defaults/main.yml b/roles/live_symlink/defaults/main.yml
index eca23ad6..51618671 100644
--- a/roles/live_symlink/defaults/main.yml
+++ b/roles/live_symlink/defaults/main.yml
@@ -6,11 +6,13 @@ live_symlink:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml
index fa1d9e97..d85e4e7f 100644
--- a/roles/live_symlink/tasks/main.yml
+++ b/roles/live_symlink/tasks/main.yml
@@ -38,7 +38,7 @@
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
     follow: false
-    force: true
+    force: "{{ link.force | default(true) }}"
   with_items: "{{ live_symlink.symlinks }}"
   loop_control:
     loop_var: link
diff --git a/roles/npm/README.md b/roles/npm/README.md
index 97f06cdc..5a17e577 100644
--- a/roles/npm/README.md
+++ b/roles/npm/README.md
@@ -18,11 +18,13 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/npm/defaults/main.yml b/roles/npm/defaults/main.yml
index a29fcac7..49321d19 100644
--- a/roles/npm/defaults/main.yml
+++ b/roles/npm/defaults/main.yml
@@ -11,11 +11,13 @@ npm:
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
-  # create: whether to create the target if it does not exists.
+  # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
   #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: true
   #  - src: '/var/simplesaml/etc'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
+  #    force: false
   symlinks: []
   # Specify any additional templates to generate, with src (template) and dest (file).
   # src: name of a template, in the "templates" dir relative to your playbook.
diff --git a/roles/npm/tasks/main.yml b/roles/npm/tasks/main.yml
index f5372b0f..09487313 100644
--- a/roles/npm/tasks/main.yml
+++ b/roles/npm/tasks/main.yml
@@ -24,6 +24,7 @@
     src: "{{ link.src }}"
     dest: "{{ deploy_path }}/{{ link.dest }}"
     state: link
+    force: "{{ link.force | default(false) }}"
   with_items: "{{ npm.symlinks }}"
   loop_control:
     loop_var: link

From 19e815c655996ca8986bc489e56c208bbc17c10c Mon Sep 17 00:00:00 2001
From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com>
Date: Tue, 20 Aug 2024 11:56:01 +0300
Subject: [PATCH 134/139] fix cachetool url (#500)

---
 roles/cli/cachetool/tasks/main.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml
index c0cbce61..c9855743 100644
--- a/roles/cli/cachetool/tasks/main.yml
+++ b/roles/cli/cachetool/tasks/main.yml
@@ -27,7 +27,7 @@
 
     - name: Download latest cachetool installer if PHP is 8.1 or newer.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -35,7 +35,7 @@
 
     - name: Download cachetool version 8.5.0 installer if PHP is 8.0.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -43,7 +43,7 @@
 
     - name: Download cachetool version 7.1.0 installer if PHP is 7.3 or newer.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -52,7 +52,7 @@
 
     - name: Download cachetool version 5.1.3 installer if PHP is 7.2.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -60,7 +60,7 @@
 
     - name: Download cachetool version 4.1.1 installer if PHP is 7.1.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -68,7 +68,7 @@
 
     - name: Download cachetool version 3.2.2 installer if PHP version is too old.
       ansible.builtin.get_url:
-        url: "http://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar"
+        url: "https://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar"
         dest: "{{ cachetool_bin }}"
         mode: 0755
       when:
@@ -76,7 +76,7 @@
 
 - name: "Download the specified {{ cachetool.version }} cachetool version installer."
   ansible.builtin.get_url:
-    url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
+    url: "https://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar"
     dest: "{{ cachetool_bin }}"
     mode: 0755
   when:

From 747ec29d8d405ddc3a04a136eb18693dd8199121 Mon Sep 17 00:00:00 2001
From: nfawbert <62660788+nfawbert@users.noreply.github.com>
Date: Thu, 26 Sep 2024 10:52:07 +0100
Subject: [PATCH 135/139] strip sandbox mode line from sqldump (#502)

---
 roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
index 0ae03be9..6b9d7dcd 100644
--- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
+++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml
@@ -184,6 +184,11 @@
   ansible.builtin.command:
     cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database `{{ mysql_sync_target_database }}`;'"
 
+- name: Remove lines containing "sandbox mode" from SQL dump.
+  ansible.builtin.shell: "sed -i '/sandbox mode/d' {{ mysql_sync_target_dump_unpacked_path }}"
+  args:
+    executable: /bin/bash
+
 - name: Repopulate database from dump.
   ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}"
   args:

From ad3165cddbbdbc3fdbd133fec820cd251c68189b Mon Sep 17 00:00:00 2001
From: drazenCE <140631110+drazenCE@users.noreply.github.com>
Date: Thu, 28 Nov 2024 16:03:57 +0100
Subject: [PATCH 136/139] Updating-crons-mailto (#506)

* Updating-crons-mailto

* Forgot-variable-drupal8
---
 roles/cron/cron_drupal7/defaults/main.yml | 4 +++-
 roles/cron/cron_drupal7/tasks/job.yml     | 1 +
 roles/cron/cron_drupal8/defaults/main.yml | 2 ++
 roles/cron/cron_drupal8/tasks/job.yml     | 1 +
 4 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml
index 419e9c4b..4dc29105 100644
--- a/roles/cron/cron_drupal7/defaults/main.yml
+++ b/roles/cron/cron_drupal7/defaults/main.yml
@@ -12,8 +12,10 @@ drupal:
           # month:
           job: cron
           # disabled: true
+          mailto: "{{ drupal.cron_mailto | default('') }}"
+  cron_mailto: ""
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
-  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
\ No newline at end of file
+  drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml
index b89a0b05..f2eb2ff7 100644
--- a/roles/cron/cron_drupal7/tasks/job.yml
+++ b/roles/cron/cron_drupal7/tasks/job.yml
@@ -31,4 +31,5 @@
     job: "{{ _cron_job_command }}"
     state: present
     disabled: "{{ entry.disabled | default(omit) }}"
+    mailto: "{{ entry.mailto if entry.mailto is defined else omit }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml
index 9d4b90f6..4dc29105 100644
--- a/roles/cron/cron_drupal8/defaults/main.yml
+++ b/roles/cron/cron_drupal8/defaults/main.yml
@@ -12,6 +12,8 @@ drupal:
           # month:
           job: cron
           # disabled: true
+          mailto: "{{ drupal.cron_mailto | default('') }}"
+  cron_mailto: ""
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml
index 7fa23c3e..77f3c6ae 100644
--- a/roles/cron/cron_drupal8/tasks/job.yml
+++ b/roles/cron/cron_drupal8/tasks/job.yml
@@ -37,4 +37,5 @@
     job: "{{ _cron_job_command }}"
     state: present
     disabled: "{{ entry.disabled | default(omit) }}"
+    mailto: "{{ entry.mailto if entry.mailto is defined else omit }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"

From dd530bbc736d64074585ce3fd9bd391c36e74640 Mon Sep 17 00:00:00 2001
From: drazenCE <140631110+drazenCE@users.noreply.github.com>
Date: Thu, 19 Dec 2024 11:57:41 +0100
Subject: [PATCH 137/139] Updating-cron-path-fixing-mailto (#508)

* Updating-cron-path-fixing-mailto

* Adding-md

* Using-cronvar-module

* Fixing-linting-condition

* Adding-mailto-dynamically-for-specific-jobs

* Fixing-incorrect-crontab-entries

* Updating-condition-for-crons

* Updating-condition-for-crons

* Updating-condition-for-crons

* Updating-condition-for-crons

* Updating-condition-for-crons

* Updating-condition-for-crons

* Updating-cron-path-fixing-mailto

* Updating-cron-path-fixing-mailto

* Updating-cron-path-fixing-mailto

* Mailto-filtering

* Polishing-cron-output

* Fixing-syntax

* Filtering-test-crons-mailto

* Testing-trim-jinja

* Cronjob-path-fix

* Reverting
---
 docs/_Sidebar.md                                   |  1 +
 docs/roles/cache_clear/cache_clear-symfony.md      | 14 ++++++++++++++
 docs/roles/cron/cron_drupal7.md                    |  1 +
 docs/roles/deploy_code.md                          |  8 ++++++++
 .../sync/database_sync/database_sync-mysql.md      |  6 +++++-
 roles/cache_clear/cache_clear-symfony/README.md    |  7 +++++++
 roles/cron/cron_drupal7/README.md                  |  1 +
 roles/cron/cron_drupal7/defaults/main.yml          |  2 --
 roles/cron/cron_drupal7/tasks/job.yml              |  3 +--
 roles/cron/cron_drupal8/defaults/main.yml          |  2 --
 roles/cron/cron_drupal8/tasks/job.yml              |  8 +-------
 roles/deploy_code/README.md                        |  8 ++++++++
 .../database_sync/database_sync-mysql/README.md    |  6 +++++-
 13 files changed, 52 insertions(+), 15 deletions(-)
 create mode 100644 docs/roles/cache_clear/cache_clear-symfony.md

diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md
index b8c4d52e..b1f28b08 100644
--- a/docs/_Sidebar.md
+++ b/docs/_Sidebar.md
@@ -10,6 +10,7 @@
        - [Matomo](/roles/cache_clear/cache_clear-matomo)
        - [Drupal 8](/roles/cache_clear/cache_clear-mautic)
        - [Opcache](/roles/cache_clear/cache_clear-opcache)
+       - [Symfony](/roles/cache_clear/cache_clear-symfony)
      - [ce-dev](/roles/ce_dev)
        - [ce-dev prebuild](/roles/ce_dev/ce_dev_prebuild)
      - [CLI Tools](/roles/cli)
diff --git a/docs/roles/cache_clear/cache_clear-symfony.md b/docs/roles/cache_clear/cache_clear-symfony.md
new file mode 100644
index 00000000..65c608f9
--- /dev/null
+++ b/docs/roles/cache_clear/cache_clear-symfony.md
@@ -0,0 +1,14 @@
+# Symfony
+Clears the Symfony application cache.
+<!--TOC-->
+<!--ENDTOC-->
+
+<!--ROLEVARS-->
+## Default variables
+```yaml
+---
+cache_clear_symfony_env: prod
+
+```
+
+<!--ENDROLEVARS-->
diff --git a/docs/roles/cron/cron_drupal7.md b/docs/roles/cron/cron_drupal7.md
index 14356d15..32c2103f 100644
--- a/docs/roles/cron/cron_drupal7.md
+++ b/docs/roles/cron/cron_drupal7.md
@@ -24,6 +24,7 @@ drupal:
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
   drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md
index 634dbbe5..b3860144 100644
--- a/docs/roles/deploy_code.md
+++ b/docs/roles/deploy_code.md
@@ -85,7 +85,15 @@ Deploying code with autoscaling clusters relies on [cloud-init](https://cloudini
 ## Default variables
 ```yaml
 ---
+# Required for NGINX config in feature branching.
+_ce_provision_data_dir: /home/deploy/ce-deploy/data
+
 deploy_code:
+  # Feature branching config.
+  feature_branch:
+    enabled: false
+    # NGINX domains to build, see ce-provision for docs: https://github.com/codeenigma/ce-provision/blob/2.x/roles/debian/nginx/defaults/main.yml#L133
+    domains: []
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md
index ab07346d..1fd2afe8 100644
--- a/docs/roles/sync/database_sync/database_sync-mysql.md
+++ b/docs/roles/sync/database_sync/database_sync-mysql.md
@@ -25,8 +25,9 @@ mysql_sync:
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
         # This can be of types:
         # - rolling: (database backups). In that case we'll need build parameters.@todo
-        # - fixed: "fixed" database name
+        # - fixed: "fixed" database name # currently this var means the same as "dump"
         # - dump: Use an existing dump. In that case, the "database" variable is the absolute file path.
+        # This parameter is ignored if var 'path_on_deploy_server' is defined.
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
@@ -37,6 +38,9 @@ mysql_sync:
         dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
         # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
         asg: ""
+        # Uncomment and specify full path if you want to use your own dump file on the deploy server (gzip compressed).
+        # In this case it's the only variable you need in 'database.source'.
+        #path_on_deploy_server: "/home/{{ deploy_user }}/compressed_database_dump.gz"
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
diff --git a/roles/cache_clear/cache_clear-symfony/README.md b/roles/cache_clear/cache_clear-symfony/README.md
index 8bb3fde2..65c608f9 100644
--- a/roles/cache_clear/cache_clear-symfony/README.md
+++ b/roles/cache_clear/cache_clear-symfony/README.md
@@ -4,4 +4,11 @@ Clears the Symfony application cache.
 <!--ENDTOC-->
 
 <!--ROLEVARS-->
+## Default variables
+```yaml
+---
+cache_clear_symfony_env: prod
+
+```
+
 <!--ENDROLEVARS-->
diff --git a/roles/cron/cron_drupal7/README.md b/roles/cron/cron_drupal7/README.md
index 14356d15..32c2103f 100644
--- a/roles/cron/cron_drupal7/README.md
+++ b/roles/cron/cron_drupal7/README.md
@@ -24,6 +24,7 @@ drupal:
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
   defer_target: ""
   drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush"
+
 ```
 
 <!--ENDROLEVARS-->
diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml
index 4dc29105..9d4b90f6 100644
--- a/roles/cron/cron_drupal7/defaults/main.yml
+++ b/roles/cron/cron_drupal7/defaults/main.yml
@@ -12,8 +12,6 @@ drupal:
           # month:
           job: cron
           # disabled: true
-          mailto: "{{ drupal.cron_mailto | default('') }}"
-  cron_mailto: ""
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml
index f2eb2ff7..ae176e11 100644
--- a/roles/cron/cron_drupal7/tasks/job.yml
+++ b/roles/cron/cron_drupal7/tasks/job.yml
@@ -1,7 +1,7 @@
 ---
 - name: Define cron job command.
   ansible.builtin.set_fact:
-    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
+    _cron_job_command: "cd {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
 
 - name: Define cron job command if deferred (ASG).
   ansible.builtin.set_fact:
@@ -31,5 +31,4 @@
     job: "{{ _cron_job_command }}"
     state: present
     disabled: "{{ entry.disabled | default(omit) }}"
-    mailto: "{{ entry.mailto if entry.mailto is defined else omit }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml
index 4dc29105..9d4b90f6 100644
--- a/roles/cron/cron_drupal8/defaults/main.yml
+++ b/roles/cron/cron_drupal8/defaults/main.yml
@@ -12,8 +12,6 @@ drupal:
           # month:
           job: cron
           # disabled: true
-          mailto: "{{ drupal.cron_mailto | default('') }}"
-  cron_mailto: ""
   # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers.
   defer: false
   # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0]
diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml
index 77f3c6ae..ffa28b2c 100644
--- a/roles/cron/cron_drupal8/tasks/job.yml
+++ b/roles/cron/cron_drupal8/tasks/job.yml
@@ -1,14 +1,9 @@
 ---
 - name: Define cron job command.
   ansible.builtin.set_fact:
-    _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
+    _cron_job_command: "cd {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
   when: deploy_operation == "deploy"
 
-- name: Revert cron job command.
-  ansible.builtin.set_fact:
-    _cron_job_command: "cd {{ previous_deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}"
-  when: deploy_operation == "revert"
-
 - name: Define cron job command if deferred (ASG).
   ansible.builtin.set_fact:
     _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\""
@@ -37,5 +32,4 @@
     job: "{{ _cron_job_command }}"
     state: present
     disabled: "{{ entry.disabled | default(omit) }}"
-    mailto: "{{ entry.mailto if entry.mailto is defined else omit }}"
   delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}"
diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md
index 634dbbe5..b3860144 100644
--- a/roles/deploy_code/README.md
+++ b/roles/deploy_code/README.md
@@ -85,7 +85,15 @@ Deploying code with autoscaling clusters relies on [cloud-init](https://cloudini
 ## Default variables
 ```yaml
 ---
+# Required for NGINX config in feature branching.
+_ce_provision_data_dir: /home/deploy/ce-deploy/data
+
 deploy_code:
+  # Feature branching config.
+  feature_branch:
+    enabled: false
+    # NGINX domains to build, see ce-provision for docs: https://github.com/codeenigma/ce-provision/blob/2.x/roles/debian/nginx/defaults/main.yml#L133
+    domains: []
   # Specify any additional symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md
index ab07346d..1fd2afe8 100644
--- a/roles/sync/database_sync/database_sync-mysql/README.md
+++ b/roles/sync/database_sync/database_sync-mysql/README.md
@@ -25,8 +25,9 @@ mysql_sync:
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"
         # This can be of types:
         # - rolling: (database backups). In that case we'll need build parameters.@todo
-        # - fixed: "fixed" database name
+        # - fixed: "fixed" database name # currently this var means the same as "dump"
         # - dump: Use an existing dump. In that case, the "database" variable is the absolute file path.
+        # This parameter is ignored if var 'path_on_deploy_server' is defined.
         type: fixed
         # For "rolling builds", so we can compute the database name.
         build_id: mybuildprod
@@ -37,6 +38,9 @@ mysql_sync:
         dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular"
         # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty.
         asg: ""
+        # Uncomment and specify full path if you want to use your own dump file on the deploy server (gzip compressed).
+        # In this case it's the only variable you need in 'database.source'.
+        #path_on_deploy_server: "/home/{{ deploy_user }}/compressed_database_dump.gz"
       target:
         database: "{{ project_name }}_dev"
         credentials_file: "/home/{{ deploy_user }}/.mysql.creds"

From bfeb10bd13091a3e1f1f1423fc2d8e3b526afed6 Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 30 Dec 2024 11:23:47 +0100
Subject: [PATCH 138/139] Fixing Mautic deployment workflow.

---
 roles/_meta/deploy-mautic/tasks/main.yml | 30 ++++++++++++------------
 roles/cron/cron_mautic/tasks/main.yml    |  7 ++++--
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/roles/_meta/deploy-mautic/tasks/main.yml b/roles/_meta/deploy-mautic/tasks/main.yml
index ab4a38d2..4ec69f31 100644
--- a/roles/_meta/deploy-mautic/tasks/main.yml
+++ b/roles/_meta/deploy-mautic/tasks/main.yml
@@ -1,37 +1,37 @@
 ---
 # Default Mautic role.
 
-- import_role:
+- ansible.builtin.import_role:
     name: _init
-- import_role:
+- ansible.builtin.import_role:
     name: deploy_code
-# - import_role:
-#     name: composer
-# - import_role:
+- ansible.builtin.import_role:
+    name: composer
+# - ansible.builtin.import_role:
 #     name: maintenance_mode
 #   vars:
 #     maintenance_mode:
 #       operation: offline
-- import_role:
+- ansible.builtin.import_role:
     name: database_backup
-- import_role:
+- ansible.builtin.import_role:
     name: config_generate
-- import_role:
+- ansible.builtin.import_role:
     name: database_apply
-- import_role:
+- ansible.builtin.import_role:
     name: live_symlink
-- import_role:
+- ansible.builtin.import_role:
     name: cache_clear/cache_clear-opcache
-- import_role:
+- ansible.builtin.import_role:
     name: cache_clear/cache_clear-mautic
-# - import_role:
+# - ansible.builtin.import_role:
 #     name: maintenance_mode
 #   vars:
 #     maintenance_mode:
 #       operation: online
-- import_role:
+- ansible.builtin.import_role:
     name: cron/cron_database_backup
-- import_role:
+- ansible.builtin.import_role:
     name: cron/cron_mautic
-- import_role:
+- ansible.builtin.import_role:
     name: _exit
diff --git a/roles/cron/cron_mautic/tasks/main.yml b/roles/cron/cron_mautic/tasks/main.yml
index f0b6071b..403536dd 100644
--- a/roles/cron/cron_mautic/tasks/main.yml
+++ b/roles/cron/cron_mautic/tasks/main.yml
@@ -1,12 +1,15 @@
 ---
+- name: Set console path.
+  ansible.builtin.set_fact:
+    symfony_console_bin: "{{ symfony_console_bin | default('env php ' + deploy_path + '/bin/console') }}"
 
 - name: Setup Mautic cron tasks.
   cron:
     name: "{{ project_name }}_{{ build_type }}_{{ entry.job }}"
     minute: "{{ entry.minute }}"
     hour: "{{ entry.hour | default(omit) }}"
-    job: "env php {{ live_symlink_dest }}/{{ webroot }}/app/console {{ entry.job }}"
+    job: "env php {{ symfony_console_bin }} {{ entry.job }}"
   with_items: "{{ cron_mautic.entries }}"
   loop_control:
     loop_var: entry
-  when: deploy_operation == 'deploy'
\ No newline at end of file
+  when: deploy_operation == 'deploy'

From cc95886cdf3c5355affe583b72e7d6e07c34babc Mon Sep 17 00:00:00 2001
From: Greg Harvey <greg.harvey@gmail.com>
Date: Mon, 30 Dec 2024 11:42:03 +0100
Subject: [PATCH 139/139] Ensuring symlink destinations exist.

---
 roles/live_symlink/defaults/main.yml |  4 ++--
 roles/live_symlink/tasks/main.yml    | 30 ++++++++++++++++++----------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/roles/live_symlink/defaults/main.yml b/roles/live_symlink/defaults/main.yml
index 51618671..8bf22492 100644
--- a/roles/live_symlink/defaults/main.yml
+++ b/roles/live_symlink/defaults/main.yml
@@ -3,11 +3,11 @@ deploy_user: 'deploy'
 live_symlink:
   # Either update (for successful builds) or revert.
   operation: update
-  # Specify any additional symlink to create, with src (target) and dest (link).
+  # Specify any additional directory symlink to create, with src (target) and dest (link).
   # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig')
   # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache')
   # force: set to true to create the symlinks in two cases: the source file does not exist but will appear later; the destination exists and is a file.
-  #  - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml'
+  #  - src: '/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/simplesaml'
   #    dest: 'vendor/simplesamlphp/simplesamlphp/config'
   #    force: true
   #  - src: '/var/simplesaml/etc'
diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml
index d85e4e7f..a60e19ea 100644
--- a/roles/live_symlink/tasks/main.yml
+++ b/roles/live_symlink/tasks/main.yml
@@ -32,16 +32,24 @@
     - live_symlink.templates | length
     - deploy_operation == 'deploy'
 
-- name: Create additional symlinks.
-  ansible.builtin.file:
-    src: "{{ link.src }}"
-    dest: "{{ deploy_path }}/{{ link.dest }}"
-    state: link
-    follow: false
-    force: "{{ link.force | default(true) }}"
-  with_items: "{{ live_symlink.symlinks }}"
-  loop_control:
-    loop_var: link
+- name: Handle additional symlinks.
   when:
     - live_symlink.symlinks | length
-    - deploy_operation == 'deploy'
\ No newline at end of file
+    - deploy_operation == 'deploy'
+  loop_control:
+    loop_var: link
+  with_items: "{{ live_symlink.symlinks }}"
+  block:
+    - name: Ensure additional symlink sources exist.
+      ansible.builtin.file:
+        path: "{{ link.src }}"
+        state: directory
+        mode: '0755'
+
+    - name: Create additional symlinks.
+      ansible.builtin.file:
+        src: "{{ link.src }}"
+        dest: "{{ deploy_path }}/{{ link.dest }}"
+        state: link
+        follow: false
+        force: "{{ link.force | default(true) }}"