From 3ccfe51727c35f7cef25216f0ae4d0701d9dabb9 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 20 Jan 2022 10:13:40 +0000 Subject: [PATCH 01/73] Defer nightly backups, disable ASG processes during syncs and run syncs with backup (#94) * Change location of nightly backup script and delegate the cron that runs it to the deploy server, if required (for ASGs). * Set up nightly backup crons in separate files in /etc/cron.d * Can't put site cron files in /etc/cron.d because the deploy user doesn't have perms. * Try and add the ability to sync a site using a nightly backup instead of a fresh DB dump. * Use Ansible modules to look up RDS host and to copy the nightly backup into place. * Delegate PATH setup in db backup cron to localhost. * Shell bad. Command good. But makes it convoluted. Oh well. * Used wrong database name in source database copy. * Try and disable the ReplaceUnhealthy auto scale process during syncs. * Fix Drupal cron roles when deferring to deploy server. --- .../cron_database_backup-mysql/tasks/main.yml | 10 +++ .../tasks/setup.yml | 18 ++++- roles/cron/cron_drupal7/tasks/main.yml | 1 + roles/cron/cron_drupal8/tasks/main.yml | 1 + .../database_sync-mysql/defaults/main.yml | 9 +++ .../database_sync-mysql/tasks/sync.yml | 79 ++++++++++++++++++- 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index 04789df8..ffd14157 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -3,6 +3,16 @@ # screwing the daily backup when using rolling db, we could # add a keep mechanism for backup scripts, like for the dumps themselves. # Nice to have more than anything. +- name: Setup PATH in crontab. + cron: + name: PATH + env: true + job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost + when: + - drupal.defer is defined + - drupal.defer + - include_tasks: setup.yml vars: database: database diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index bdc1534c..40b0c47e 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -25,12 +25,26 @@ - name: Create backup script. template: src: "regular-backups.sh.j2" - dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" mode: 0700 +- name: Define backup cron job command. + set_fact: + _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + +- name: Define backup cron job command if deferred (ASG). + set_fact: + _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\"" + when: + - drupal.defer is defined + - drupal.defer + - drupal.defer_target is defined + - drupal.defer_target | length > 0 + - name: Setup regular backup for MySQL. cron: name: "cron_mysql_{{ database.host }}_{{ database.original.database }}" minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" - job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + job: "{{ _backup_cron_job_command }}" + delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index c1d7b259..48e3e4a0 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -15,6 +15,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -25,3 +32,5 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 81e9b2ce..b8182bc3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -1,4 +1,40 @@ --- +- name: Get database source host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_source_database_host_info + delegate_to: "{{ database.source.host }}" + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. +- name: Get database target host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_target_database_host_info + when: + - database.target.asg is defined + - database.target.asg | length > 0 + +- name: Disable ReplaceUnhealthy autoscale process on source ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Disable ReplaceUnhealthy autoscale process on target ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 + - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" @@ -33,7 +69,30 @@ - name: Take a dump from source database. shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" - when: not database.source.type == 'dump' + when: + - not database.source.type == 'dump' + - database.source.fresh_db + +- name: Find source database host. + ansible.builtin.command: + cmd: "grep 'host' {{ database.source.credentials_file }}" + register: mysql_host_info_grep + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Register source database host. + set_fact: + mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Copy a nightly backup for the source database. + ansible.builtin.copy: + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" + dest: "{{ mysql_sync_source_dump_path }}" + remote_src: true + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db - name: Register tmp target dump name. set_fact: @@ -95,3 +154,21 @@ path: "{{ mysql_sync_target_dump_path }}" state: absent when: not database.target.type == 'dump' + +- name: Enable all autoscale processes on source ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Enable all autoscale processes on target ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 From a54c09cb0ce5b6889e672ba8ea741b71a3da034b Mon Sep 17 00:00:00 2001 From: Dionisio Date: Thu, 10 Mar 2022 14:16:13 +0100 Subject: [PATCH 02/73] Added deploy.yml examples for Drupal 9 and Localgov. Updated tests (#97) --- .github/workflows/ce-deploy-test.yml | 22 ++++++++ ce-dev/ansible/examples/drupal9/deploy.yml | 55 +++++++++++++++++++ ce-dev/ansible/examples/localgov/deploy.yml | 59 +++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 ce-dev/ansible/examples/drupal9/deploy.yml create mode 100644 ce-dev/ansible/examples/localgov/deploy.yml diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml index baa13347..141fc8e7 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-test.yml @@ -59,6 +59,28 @@ jobs: curl https://www.test.local shell: bash + - name: Run a test ce-dev deploy with Drupal 9 + run: | + ce-dev create -p testnine -t drupal9 -d ~/testnine + cd ~/testnine + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testnine.local + shell: bash + + - name: Run a test ce-dev deploy with Localgov + run: | + ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov + cd ~/testlocalgov + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testlocalgov.local + shell: bash + # Builds the table of contents for the docs - name: Documentation (build table of contents) if: ${{ github.event.pull_request.base.ref == '1.x' }} diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml new file mode 100644 index 00000000..deeef9ee --- /dev/null +++ b/ce-dev/ansible/examples/drupal9/deploy.yml @@ -0,0 +1,55 @@ +--- +# Template playbook for a local Drupal9 codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: no + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml new file mode 100644 index 00000000..9844206b --- /dev/null +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -0,0 +1,59 @@ +--- +# Template playbook for a local localgov codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si localgov" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: false + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + - name: Install localgov. + command: + cmd: composer require localgovdrupal/localgov + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file From 223a3b5ecec42997d9e52de690cf7b8c5f9f19ab Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 31 Mar 2022 12:43:36 +0200 Subject: [PATCH 03/73] Adding a new SimpleSAMLphp meta role. (#100) --- roles/_meta/deploy-simplesamlphp/README.md | 9 +++++++++ roles/_meta/deploy-simplesamlphp/tasks/main.yml | 17 +++++++++++++++++ .../deploy_code-simplesamlphp/tasks/main.yml | 1 + 3 files changed, 27 insertions(+) create mode 100644 roles/_meta/deploy-simplesamlphp/README.md create mode 100644 roles/_meta/deploy-simplesamlphp/tasks/main.yml create mode 100644 roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml diff --git a/roles/_meta/deploy-simplesamlphp/README.md b/roles/_meta/deploy-simplesamlphp/README.md new file mode 100644 index 00000000..3d82e671 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/README.md @@ -0,0 +1,9 @@ +# SimpleSAMLphp +Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer. + +This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision: + +* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx + + + diff --git a/roles/_meta/deploy-simplesamlphp/tasks/main.yml b/roles/_meta/deploy-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..7af28fa2 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Default SimpleSAMLphp role. This is suitable for a standalone SimpleSAMLphp installation + +- ansible.builtin.import_role: + name: _init +- ansible.builtin.import_role: + name: deploy_code +- ansible.builtin.import_role: + name: composer +- ansible.builtin.import_role: + name: database_backup +- ansible.builtin.import_role: + name: live_symlink +- ansible.builtin.import_role: + name: cache_clear/cache_clear-opcache +- ansible.builtin.import_role: + name: _exit diff --git a/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..03c03856 --- /dev/null +++ b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml @@ -0,0 +1 @@ +# Nothing to do. \ No newline at end of file From 32843d53a625b68bc3a3f7a8cfcced5c90df3f49 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 13 Apr 2022 13:09:01 +0200 Subject: [PATCH 04/73] Allowing users to set cachetool version properly. (#102) --- roles/cli/cachetool/defaults/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index ad2c6711..5ec1b2d7 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: 8.2.2 \ No newline at end of file + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version \ No newline at end of file diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 5a1969c6..aa27df04 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -13,7 +13,7 @@ when: - deploy_operation == 'deploy' -- name: Download cachetool installer. +- name: Download latest cachetool installer. get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" dest: "{{ cachetool_bin }}" @@ -21,3 +21,14 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists + - cachetool.version == 'latest' + +- name: "Download cachetool version {{ cachetool.version }} installer." + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - deploy_operation == 'deploy' + - not cachetool_global.stat.exists + - cachetool.version != 'latest' From 91ea64efc91984ce88138857898b927b5ffd7b25 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 14 Apr 2022 17:13:09 +0200 Subject: [PATCH 05/73] Deploy ami pr 1.x (#106) * Including an ami.yml for packing AMIs on build finale. * New api_call role, focused on GitLab for now. * Optionally trigger an infra build with an API call. --- roles/api_call/README.md | 9 ++++++++ roles/api_call/defaults/main.yml | 13 +++++++++++ roles/api_call/tasks/main.yml | 35 +++++++++++++++++++++++++++++ roles/deploy_code/defaults/main.yml | 21 +++++++++++++++++ roles/deploy_code/tasks/cleanup.yml | 27 +++++++++++++++++----- roles/deploy_code/tasks/deploy.yml | 10 ++++----- roles/deploy_code/tasks/main.yml | 2 +- 7 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 roles/api_call/README.md create mode 100644 roles/api_call/defaults/main.yml create mode 100644 roles/api_call/tasks/main.yml diff --git a/roles/api_call/README.md b/roles/api_call/README.md new file mode 100644 index 00000000..83d1ab28 --- /dev/null +++ b/roles/api_call/README.md @@ -0,0 +1,9 @@ +# API call + +Making RESTful API calls to other platforms. + + + + + + diff --git a/roles/api_call/defaults/main.yml b/roles/api_call/defaults/main.yml new file mode 100644 index 00000000..091342e8 --- /dev/null +++ b/roles/api_call/defaults/main.yml @@ -0,0 +1,13 @@ +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 \ No newline at end of file diff --git a/roles/api_call/tasks/main.yml b/roles/api_call/tasks/main.yml new file mode 100644 index 00000000..c837df2d --- /dev/null +++ b/roles/api_call/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Ensure variables are empty. + ansible.builtin.set_fact: + _api_call_variables: "" + _api_call_url: "" + +- name: Build HTML escaped variable string. + ansible.builtin.set_fact: + _api_call_variables: "{{ _api_call_variables + ('' if ansible_loop.first else '&') + 'variables' + item }}" + with_items: "{{ api_call.variables }}" + loop_control: + extended: true + when: api_call.variables | length > 0 + +- name: Build anonymous API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{{ _api_call_variables }}" + when: api_call.token | length == 0 + +- name: Build token authenticated API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{% if api_call.token_type == 'trigger' %}token={% else %}private_token={% endif %}{{ api_call.token }}&{{ _api_call_variables }}" + when: api_call.token | length > 0 + +- name: Display URL to call. + ansible.builtin.debug: + msg: "{{ _api_call_url }}" + +- name: Make API call. + ansible.builtin.uri: + url: "{{ _api_call_url }}" + method: "{{ api_call.method }}" + return_content: true + status_code: "{{ api_call.status_codes }}" + register: _api_call_return diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index 19e45eee..d616bf08 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -21,3 +21,24 @@ deploy_code: # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index dc72c752..4e648d25 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -1,13 +1,13 @@ --- - name: Ensure codebase is writable. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on directory. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: @@ -15,13 +15,13 @@ - deploy_code.perms_fix_path | length > 1 - name: Delete codebases. - file: + ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} - name: Create a tarball of the deployed codebases. - command: + ansible.builtin.command: cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined @@ -29,7 +29,7 @@ run_once: true - name: Create destination folder. - file: + ansible.builtin.file: path: "{{ deploy_code.mount_sync }}" state: directory mode: "0755" @@ -39,9 +39,24 @@ run_once: true - name: Move to final destination. - command: + ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 run_once: true + +- name: Trigger an infrastructure rebuild. + ansible.builtin.include_role: + name: api_call + vars: + api_call: + type: "{{ deploy_code.api_call.type }}" + base_url: "{{ deploy_code.api_call.base_url }}" + path: "{{ deploy_code.api_call.path }}" + method: "{{ deploy_code.api_call.method }}" + token: "{{ deploy_code.api_call.token }}" + token_type: "{{ deploy_code.api_call.token_type }}" + variables: "{{ deploy_code.api_call.variables }}" + status_codes: "{{ deploy_code.api_call.status_codes }}" + when: deploy_code.rebuild_infra diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml index 3693eb70..0d15425c 100644 --- a/roles/deploy_code/tasks/deploy.yml +++ b/roles/deploy_code/tasks/deploy.yml @@ -1,6 +1,6 @@ --- - name: Copy project repository. - synchronize: + ansible.posix.synchronize: src: "{{ _ce_deploy_build_dir }}/" dest: "{{ deploy_path }}" archive: true @@ -8,17 +8,17 @@ - "--exclude=.git" - name: Ensure project repository is readable. - file: + ansible.builtin.file: path: "{{ deploy_path }}" state: directory mode: 0755 - name: Project specific tasks. - include_role: + ansible.builtin.include_role: name: "deploy_code/deploy_code-{{ project_type }}" - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ deploy_code.templates }}" @@ -29,7 +29,7 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link diff --git a/roles/deploy_code/tasks/main.yml b/roles/deploy_code/tasks/main.yml index a8021dcf..f4bd05af 100644 --- a/roles/deploy_code/tasks/main.yml +++ b/roles/deploy_code/tasks/main.yml @@ -1,2 +1,2 @@ --- -- include_tasks: "{{ deploy_operation }}.yml" +- ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" From 371b6b9d8b28a908facfe2d80779efd76cb64b40 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 16:20:36 +0200 Subject: [PATCH 06/73] Making the MySQL dump command for routine back-ups less aggressive. (#104) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. --- roles/_init/defaults/main.yml | 1 + .../cron_database_backup-mysql/defaults/main.yml | 1 + .../templates/regular-backups.sh.j2 | 5 +++-- .../database_backup/database_backup-mysql/defaults/main.yml | 1 + .../database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- .../sync/database_sync/database_sync-mysql/defaults/main.yml | 1 + roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 10 insertions(+), 5 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 06470a09..9821f131 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,6 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" +_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index c6c57269..29b8003b 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,3 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index 901cfcf7..fb60cdea 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,8 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - mysqldump -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" + {{ cron_mysql_backup.mysqldump_command }} \ + -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } @@ -23,4 +24,4 @@ cleanup(){ } backup -cleanup \ No newline at end of file +cleanup diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 3ad97b6c..09f44ca3 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,6 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 4b470b89..57a46e2e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index fe32089a..cbade053 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 48e3e4a0..008f0cf4 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,5 +1,6 @@ --- mysql_sync: + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index b8182bc3..778092bc 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From a9cc7adf64a8c7959f971df84df9e6172d0cf4c6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 18:18:47 +0200 Subject: [PATCH 07/73] Fix database backups pr 1.x (#109) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. * Defaults file must be the first param for mysqldump. --- roles/_init/defaults/main.yml | 2 +- .../cron_database_backup-mysql/defaults/main.yml | 2 +- .../cron_database_backup-mysql/templates/regular-backups.sh.j2 | 2 +- roles/database_backup/database_backup-mysql/defaults/main.yml | 2 +- .../database_backup/database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- roles/sync/database_sync/database_sync-mysql/defaults/main.yml | 2 +- roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9821f131..c4d15b78 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,7 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" -_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index 29b8003b..f0c869f3 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,4 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index fb60cdea..33c27a5a 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,7 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - {{ cron_mysql_backup.mysqldump_command }} \ + mysqldump {{ cron_mysql_backup.mysqldump_params }} \ -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 09f44ca3..f7e070e9 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,7 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 57a46e2e..d06b1543 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index cbade053..1db841b0 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 008f0cf4..0ab99e98 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,6 @@ --- mysql_sync: - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 778092bc..c3ff0b9e 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From 4c4c3ad1f9ecd25e25e9c2e5f79d4cb8108c9f34 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 22 Apr 2022 10:28:11 +0100 Subject: [PATCH 08/73] Fix MySQL backup deferral. (#110) --- .../cron_database_backup-mysql/tasks/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index 40b0c47e..d09fbbac 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -47,4 +47,4 @@ minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" - delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" + delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}" From 029848d1163191087bd85f372973594b283aad17 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 26 Apr 2022 11:30:31 +0100 Subject: [PATCH 09/73] Files recurse fix pr 1.x (#112) * Don't recurse through site directory when setting permissions during config_generate step. * Update drupal7 config_generate perms update task and use true/false instead of yes/no. --- .../config_generate/config_generate-drupal7/tasks/settings.yml | 3 +-- .../config_generate/config_generate-drupal8/tasks/settings.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f82c9aa7..277899a0 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -15,9 +15,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index a9937d8e..70d9b113 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -19,9 +19,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: From b71fa83d6b4deda1daa4532c3a00e3835118cf4a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 29 Apr 2022 11:52:28 +0100 Subject: [PATCH 10/73] Improve multisite support (#115) * Pass -l option in drush commands to specify site name, which should match the site folder name. * Move -l option in drush commands to the first option instead of the last. --- roles/cache_clear/cache_clear-drupal7/tasks/main.yml | 2 +- roles/cache_clear/cache_clear-drupal8/tasks/main.yml | 2 +- .../database_apply-drupal7/tasks/features.yml | 4 ++-- roles/database_apply/database_apply-drupal7/tasks/main.yml | 4 ++-- roles/database_apply/database_apply-drupal8/tasks/main.yml | 6 +++--- .../maintenance_mode-drupal-core/tasks/offline.yml | 4 ++-- .../maintenance_mode-drupal-core/tasks/online.yml | 4 ++-- .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 4 ++-- .../admin_creds/admin_creds-drupal8/tasks/admin.yml | 4 ++-- roles/sanitize/sanitize-drupal8/tasks/main.yml | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml index 1261ff4b..6f72b3a5 100644 --- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal 7 cache. command: - cmd: "{{ drush_bin }} -y cc all" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index cfd7fdd3..5b780086 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal cache. command: - cmd: "{{ drush_bin }} -y cr" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index 19d58e9b..edba46bf 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,8 +1,8 @@ --- - name: Check if Features module is enabled. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} pm-info features | grep ': enabled' | wc -l" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" register: features_enabled - name: Revert Drupal configuration from Features. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -y {{ site.revert_features_command }}" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}" when: features_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 048a1457..7c2201e2 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -21,7 +21,7 @@ - name: Apply Drupal database updates. shell: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" loop_control: diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 5634faca..dc138585 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -15,7 +15,7 @@ - name: Install Drupal. command: - cmd: "{{ drush_bin }} {{ site.install_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -34,7 +34,7 @@ - name: Apply Drupal database updates. command: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -44,7 +44,7 @@ - name: Import configuration. command: - cmd: "{{ drush_bin }} -y {{ site.config_import_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 2872226a..cbe65b07 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -2,7 +2,7 @@ # @todo this needs refactoring for multisite. - name: Enable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 1 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -12,7 +12,7 @@ - name: Enable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 1" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 99057084..e6a9cbf5 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,7 +1,7 @@ --- - name: Disable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 0 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -10,7 +10,7 @@ - name: Disable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 0" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml index 98401c5c..c8d75bd7 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml @@ -6,6 +6,6 @@ _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}" - name: Reset admin username. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" - name: Reset admin password. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml index 0302308b..c2adafdf 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml @@ -4,14 +4,14 @@ # Loading the user directly is akward, but at least means we don't bypass entity update. - name: Reset admin username. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.username | length > 1 - name: Reset admin password. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.password | length > 1 diff --git a/roles/sanitize/sanitize-drupal8/tasks/main.yml b/roles/sanitize/sanitize-drupal8/tasks/main.yml index 0a33f5a8..26675f27 100644 --- a/roles/sanitize/sanitize-drupal8/tasks/main.yml +++ b/roles/sanitize/sanitize-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Sanitize database. command: - cmd: "{{ drush_bin }} -y {{ site.sanitize_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.sanitize_command }}" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" From c933dc8224e42106df909e29cdb8928f5c5da580 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 23 May 2022 16:19:31 +0100 Subject: [PATCH 11/73] Static credentials handling fix pr 1.x (#119) * Attempt to fix static credentials handling when deploying to an ASG. * Include build_type in static password file location. * Move to using new location for static credentials handling, but try to catch any legacy stuff too, for now. * Add a couple of comments to legacy handling of static credentials. --- .../database_backup-mysql/tasks/deploy.yml | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index a67d240e..dbb39de7 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -24,8 +24,55 @@ - set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' + +### Start of legacy handling. Can remove in a few months. +- name: Catch legacy static password handling. + ansible.builtin.stat: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + register: _legacy_static_creds + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Create new static password location if it doesn't exist. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0755 + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Copy legacy static password to new location. + ansible.builtin.copy: + src: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + dest: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0644 + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists + +- name: Delete legacy static password file if it exists. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + state: absent + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists +### End of legacy handling. + - set_fact: - _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" + _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. - set_fact: From b6ea0206e8f83f7e2459babd0622403b955276af Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 31 May 2022 16:26:10 +0200 Subject: [PATCH 12/73] Making contents of deploy tar 'ownerless'. (#117) --- roles/deploy_code/tasks/cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 4e648d25..03530f22 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -22,7 +22,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 1c2b206d009b21d10b96c434168c15ac5700b96c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 7 Jun 2022 15:48:04 +0100 Subject: [PATCH 13/73] Implement file syncing (#124) * Add a files_sync role. * Clear up some comments, add temp_dir variable and use rsync with command module instead of synchronize module, in files_sync role. * Need trailing slash on src when syncing files to destination server so the contents get synced and not the parent directory itself. * Use rsync instead of copy when syncing files to target server as copy is so slow. * Ensure file sync tasks are run only once. --- roles/sync/files_sync/defaults/main.yml | 16 ++++++++++++++++ roles/sync/files_sync/tasks/main.yml | 7 +++++++ roles/sync/files_sync/tasks/sync.yml | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 roles/sync/files_sync/defaults/main.yml create mode 100644 roles/sync/files_sync/tasks/main.yml create mode 100644 roles/sync/files_sync/tasks/sync.yml diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml new file mode 100644 index 00000000..5b65b1fb --- /dev/null +++ b/roles/sync/files_sync/defaults/main.yml @@ -0,0 +1,16 @@ +--- +files_sync: + directories: + - source: + # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files" + # Host that contains source files. + host: "localhost" + # Location on deploy server where source files get copied to first. NO TRAILING SLASH. + temp_dir: "/tmp" + # Used to create directory in /tmp. + build_id: mybuildprod + target: + # Location of the files to sync to. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files" + build_id: mybuilddev diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml new file mode 100644 index 00000000..47c3667a --- /dev/null +++ b/roles/sync/files_sync/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Sync files. + include_tasks: "sync.yml" + with_items: "{{ files_sync.directories }}" + loop_control: + loop_var: files + run_once: true diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml new file mode 100644 index 00000000..e2160008 --- /dev/null +++ b/roles/sync/files_sync/tasks/sync.yml @@ -0,0 +1,21 @@ +--- +- name: Create a temporary directory for source files on localhost. + ansible.builtin.file: + path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + delegate_to: localhost + run_once: true + +- name: Copy the source files onto the deploy server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" + delegate_to: "localhost" + run_once: true + +- name: Copy the source files from the deploy server onto the destination server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: "localhost" + run_once: true From 1cd0ed0797c1d26a933b4a3e06e0279e4af6a05a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Wed, 8 Jun 2022 12:47:34 +0100 Subject: [PATCH 14/73] Create Drupal-specific sync roles (#128) * Add a drupal_sync_tasks role to run Drupal specific tasks during syncs. * Move sync database_apply files into tasks subdirectory. * Woops, used bad role names in sync database update roles. * Remove feature reverting from drupal7 database_apply sync role. --- .../cache_clear-drupal7/tasks/main.yml | 11 ++++++++ .../cache_clear-drupal8/tasks/main.yml | 11 ++++++++ .../database_apply-drupal7/tasks/main.yml | 12 +++++++++ .../database_apply-drupal8/tasks/main.yml | 26 +++++++++++++++++++ .../database_apply/tasks/main.yml | 6 +++++ 5 files changed, 66 insertions(+) create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml new file mode 100644 index 00000000..ab9342ba --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal 7 cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml new file mode 100644 index 00000000..96c721ee --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml new file mode 100644 index 00000000..f520514e --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.shell: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml new file mode 100644 index 00000000..bb9cc73c --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Import configuration. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - site.config_import_command + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml new file mode 100644 index 00000000..58f6215a --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Apply database updates. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/database_apply/database_apply-{{ project_type }}" + when: deploy_operation == 'deploy' + run_once: true From 904033f8bf3e1973d7b78c8ea2072426f1d0decb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 10 Jun 2022 12:57:40 +0200 Subject: [PATCH 15/73] Fixing GRANT query for MySQL > 8.0. (#131) * Fixing GRANT query for MySQL > 8.0. * Create what though? --- roles/_init/defaults/main.yml | 1 + roles/database_backup/database_backup-mysql/tasks/deploy.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index c4d15b78..790e9409 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -1,6 +1,7 @@ --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index dbb39de7..b0328958 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -88,8 +88,9 @@ # to allow this role to be looped over, # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. +# As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 94715d8bd5363c640649e09bffe0e3b1574ceb90 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 13 Jun 2022 10:34:12 +0100 Subject: [PATCH 16/73] Use IF NOT EXISTS when creating database user as that command fails if static database users are being used. (#133) --- roles/database_backup/database_backup-mysql/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index b0328958..f5a7130a 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -90,7 +90,7 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 42f12c81d0974648857d203139a154c736ac9731 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 16 Jun 2022 13:39:30 +0100 Subject: [PATCH 17/73] Attempt to fix syncs whenever the 'dump' type is used for source or target. (#149) --- .../database_sync-mysql/tasks/sync.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c3ff0b9e..98eca0f3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -38,12 +38,6 @@ - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" - when: not database.source.type == 'dump' - -- name: Register existing remote dump name. - set_fact: - mysql_sync_source_dump_path: "{{ database.source.database }}" - when: database.source.type == 'dump' - name: Get source last known good build number. command: @@ -70,7 +64,6 @@ shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - - not database.source.type == 'dump' - database.source.fresh_db - name: Find source database host. @@ -97,12 +90,6 @@ - name: Register tmp target dump name. set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" - when: not database.target.type == 'dump' - -- name: Register final target dump name. - set_fact: - mysql_sync_target_dump_path: "{{ database.target.database }}" - when: database.target.type == 'dump' - name: Get target last known good build number. command: @@ -139,21 +126,17 @@ - name: Drop target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Recreate target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Repopulate database from dump. shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - when: not database.target.type == 'dump' - name: Remove tmp dump file. file: path: "{{ mysql_sync_target_dump_path }}" state: absent - when: not database.target.type == 'dump' - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > From a9c6abfb5c686d030a0671d8a20805b7d3b1be02 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 20 Jun 2022 11:31:01 +0200 Subject: [PATCH 18/73] Squashfs pr 1.x (#150) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. --- roles/_init/tasks/main.yml | 51 +++++++++---- roles/deploy_code/defaults/main.yml | 13 +++- roles/deploy_code/tasks/cleanup.yml | 108 +++++++++++++++++++++++++++- roles/live_symlink/tasks/main.yml | 23 ++++-- 4 files changed, 171 insertions(+), 24 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a1b6e204..53be27d3 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,27 +1,45 @@ --- # Ensure default values for common variables. - name: Define deploy user. - set_fact: + ansible.builtin.set_fact: deploy_user: "{{ deploy_user | default('deploy') }}" - name: Define deploy base path. - set_fact: + ansible.builtin.set_fact: deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}" - name: Define mounted directory for assets. - set_fact: + ansible.builtin.set_fact: deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}" - name: Define webroot. - set_fact: + ansible.builtin.set_fact: webroot: "{{ webroot | default('web') }}" - name: Define build deploy path prefix. - set_fact: + ansible.builtin.set_fact: deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_" - name: Define build deploy path. - set_fact: + ansible.builtin.set_fact: deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}" - name: Define live_symlink dest. - set_fact: + ansible.builtin.set_fact: live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" +# Manipulate variables for SquashFS builds. +- name: Define image builds base path. + ansible.builtin.set_fact: + build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" + when: deploy_code.mount_type == "squashfs" +- name: Define image builds build path prefix. + ansible.builtin.set_fact: + build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" + when: deploy_code.mount_type == "squashfs" +- name: Define live_symlink dest for image builds. + ansible.builtin.set_fact: + live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" + when: deploy_code.mount_type == "squashfs" +- name: Overwrite deploy and live_symlink paths if SquashFS deploy. + ansible.builtin.set_fact: + deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" + when: deploy_code.mount_type == "squashfs" + # Gather last known good build directly from symlink. # This can happen: # - when the first builds failed, @@ -42,7 +60,7 @@ # This is passed from caller. - name: Gather last known good build number. - set_fact: + ansible.builtin.set_fact: previous_build_number: "{{ previous_known_build_number }}" # - set_fact: @@ -53,27 +71,34 @@ # Make sure the deploy target exists. - name: Ensure deploy target directory exists. - file: + ansible.builtin.file: path: "{{ deploy_base_path }}" state: directory +# Make sure the build target exists. +- name: Ensure build target directory exists. + ansible.builtin.file: + path: "{{ build_base_path }}" + state: directory + when: deploy_code.mount_type == "squashfs" + # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. - stat: + ansible.builtin.stat: path: "{{ role_path }}/tasks/{{ project_type }}.yml" register: _project_type_task_result delegate_to: "localhost" # Project specific init tasks. - name: Include project init tasks. - include_tasks: "{{ project_type }}.yml" + ansible.builtin.include_tasks: "{{ project_type }}.yml" when: - _project_type_task_result.stat.exists - name: Define opcache cachetool path. - set_fact: + ansible.builtin.set_fact: cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}" - name: Ensure we have a cachetool binary. - import_role: + ansible.builtin.import_role: name: cli/cachetool diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index d616bf08..f1f7cdb4 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -3,7 +3,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -15,12 +15,21 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false # Details of API call to trigger. See api_call role. diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 03530f22..c722b049 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -6,19 +6,39 @@ become: true when: "www_user != deploy_user" -- name: Ensure permissions are set on directory. +- name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type != "squashfs" -- name: Delete codebases. +- name: Ensure permissions are set on builds directory. + ansible.builtin.shell: + cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.perms_fix_path is defined + - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Delete codebases from deploy directory. ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type != "squashfs" + +- name: Delete codebases from builds directory. + ansible.builtin.file: + name: "{{ build_path_prefix }}{{ item }}" + state: absent + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type == "squashfs" - name: Create a tarball of the deployed codebases. ansible.builtin.command: @@ -26,6 +46,26 @@ when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Ensure older SquashFS images are deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + run_once: true + +- name: Create a SquashFS image of the deployed codebases. + ansible.builtin.command: + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true - name: Create destination folder. @@ -38,14 +78,76 @@ - deploy_code.mount_sync | length > 1 run_once: true -- name: Move to final destination. +- name: Move tar file to final destination. ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Move SquashFS image to final destination. + ansible.builtin.command: + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true +- name: Copy SquashFS image to local server. + ansible.builtin.command: + cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Check if we have a mount already. + ansible.builtin.shell: + cmd: "mount | grep {{ deploy_base_path }}" + ignore_errors: true + register: _mount_check + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount -l {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + +- name: Reload any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: reloaded + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + - deploy_code.services | length > 0 + +- name: Mount new SquashFS image. + ansible.builtin.command: + cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: name: api_call diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml index 460ced9a..fa1d9e97 100644 --- a/roles/live_symlink/tasks/main.yml +++ b/roles/live_symlink/tasks/main.yml @@ -1,19 +1,28 @@ --- -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ previous_build_number }}" when: deploy_operation == 'revert' +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_dest }}" + +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_build_dest }}" + when: deploy_code.mount_type == "squashfs" + - name: Symlink build. - file: + ansible.builtin.file: src: "{{ _live_symlink_build_target }}" - dest: "{{ live_symlink_dest }}" + dest: "{{ _live_symlink_dest_target }}" state: link + follow: false + force: true - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ live_symlink.templates }}" @@ -24,10 +33,12 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link + follow: false + force: true with_items: "{{ live_symlink.symlinks }}" loop_control: loop_var: link From a4de7710d2d7609aa9213c1f73951eaf27b23cbd Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 24 Jun 2022 13:00:24 +0100 Subject: [PATCH 19/73] Check deploy_code.mount_type is defined when setting facts in init role. (#155) --- roles/_init/tasks/main.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 53be27d3..effa2790 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -26,19 +26,27 @@ - name: Define image builds base path. ansible.builtin.set_fact: build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define image builds build path prefix. ansible.builtin.set_fact: build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define live_symlink dest for image builds. ansible.builtin.set_fact: live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Overwrite deploy and live_symlink paths if SquashFS deploy. ansible.builtin.set_fact: deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Gather last known good build directly from symlink. # This can happen: @@ -80,7 +88,9 @@ ansible.builtin.file: path: "{{ build_base_path }}" state: directory - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. From fe2aaf10f5bfc9124f5c1ab1cb6bff6e8e5b635c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 28 Jun 2022 11:08:07 +0100 Subject: [PATCH 20/73] Make config imports during syncs optional (#157) * Use a different variable for config imports during a sync so they can be optional. * Actually, no. Use a variable to determine if config should be imported during a sync. * Fix some logic with config import during syncs and add a comment to explain when the task runs. --- .../database_apply/database_apply-drupal8/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index bb9cc73c..e050af3f 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -9,6 +9,7 @@ loop_control: loop_var: site +# This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" @@ -20,6 +21,7 @@ loop_var: site when: - site.config_import_command + - site.sync_config_import is not defined or site.sync_config_import - name: Clear the cache. ansible.builtin.include_role: From 451227d28461c0afcc0bb854277fda17796d1d9e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jun 2022 17:40:45 +0200 Subject: [PATCH 21/73] Squashfs pr 1.x (#153) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. * Adding a revert behaviour for SquashFS. * Making some wording a little less ambiguous. * Fixing bad image path and adding clauses to stat check. * Trying to make remounting SquashFS images a bit safer. * Adding when clauses to SquashFS image path check. * Removing revert code, as it cannot work. --- roles/deploy_code/tasks/cleanup.yml | 64 +++++++++++++++++++---------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index c722b049..e1412749 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -49,16 +49,6 @@ - deploy_code.mount_type == "tarball" run_once: true -- name: Ensure older SquashFS images are deleted. - ansible.builtin.file: - path: "{{ build_base_path }}/deploy.sqsh" - state: absent - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - run_once: true - - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" @@ -96,6 +86,36 @@ - deploy_code.mount_type == "squashfs" run_once: true +- name: Check if we have a SquashFS image already there. + ansible.builtin.stat: + path: "{{ build_base_path }}/deploy.sqsh" + register: _deploy_code_mount_image + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Copy previous SquashFS image in case of rollback. + ansible.builtin.copy: + remote_src: true + force: true + src: "{{ build_base_path }}/deploy.sqsh" + dest: "{{ build_base_path }}/deploy_previous.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_image.stat.islnk is defined + +- name: Ensure mounted SquashFS image is deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Copy SquashFS image to local server. ansible.builtin.command: cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" @@ -108,21 +128,11 @@ ansible.builtin.shell: cmd: "mount | grep {{ deploy_base_path }}" ignore_errors: true - register: _mount_check - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Unmount existing SquashFS image. - ansible.builtin.command: - cmd: "umount -l {{ deploy_base_path }}" - become: true + register: _deploy_code_mount_check when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded - name: Reload any services that might be keeping the loop device busy. ansible.builtin.service: @@ -136,9 +146,19 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded + - _deploy_code_mount_check is succeeded - deploy_code.services | length > 0 +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - name: Mount new SquashFS image. ansible.builtin.command: cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" From c7aae71b8903846fccbbd325464509282cb6a302 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 16 Aug 2022 12:18:58 +0100 Subject: [PATCH 22/73] Add cache clears to Drupal deployments, before DB updates and stuff (#159) * Add cache clears to Drupal deployments, before database updates and config import are run. * Move cache clears out of meta roles and only run in database_apply role on non-initial builds. * Add cache clear stuff to drupal7 database_apply role. --- roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++ roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 7c2201e2..8b655bda 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -19,6 +19,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index dc138585..23cb0b3b 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -32,6 +32,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 15b8daae0b9736f2a80878415a02155dd2acc8e0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 5 Sep 2022 18:05:10 +0200 Subject: [PATCH 23/73] Avoid leaving exponentially growing sqsh files in build locations! (#164) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. --- roles/deploy_code/tasks/cleanup.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index e1412749..b206b7ab 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -95,17 +95,18 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" -- name: Copy previous SquashFS image in case of rollback. +- name: Copy previous SquashFS image to shared mount point in case of rollback. ansible.builtin.copy: remote_src: true force: true src: "{{ build_base_path }}/deploy.sqsh" - dest: "{{ build_base_path }}/deploy_previous.sqsh" + dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - _deploy_code_mount_image.stat.islnk is defined + run_once: true - name: Ensure mounted SquashFS image is deleted. ansible.builtin.file: From 5cab4ae00d301e183ab1d17cf53bf289391b4a6b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 6 Oct 2022 16:28:30 +0200 Subject: [PATCH 24/73] Exclude sqsh file pr 1.x (#167) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. * Making sure image/tarball filenames are build specific in /tmp. --- roles/deploy_code/tasks/cleanup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index b206b7ab..353854b3 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -42,7 +42,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -70,7 +70,7 @@ - name: Move tar file to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -79,7 +79,7 @@ - name: Move SquashFS image to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 459faba5e163ee6355d4bb56f9ad19042c7aa148 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 7 Oct 2022 16:05:35 +0200 Subject: [PATCH 25/73] Removing unnecessary lines in Drupal config generation. (#169) * Removing unnecessary lines in Drupal config generation. * Drupal 7 lacked the install_command var. --- .../tasks/settings.yml | 16 ++++++--------- .../tasks/settings.yml | 20 ++++++++----------- .../database_apply-drupal7/tasks/main.yml | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index 277899a0..f8e3b55e 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -1,25 +1,21 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set private files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index 70d9b113..ebf182e8 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -1,29 +1,25 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" -- name: set private files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set config directory path. - set_fact: +- name: Set config directory path. + ansible.builtin.set_fact: build_config_sync_directory: "{{ deploy_path }}/{{ site.config_sync_directory }}" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 8b655bda..eb2edbfa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -45,4 +45,4 @@ include_tasks: ctools.yml with_items: "{{ drupal.sites }}" loop_control: - loop_var: site \ No newline at end of file + loop_var: site From 200f59002c0459d63edc4d821f7b31ee94dde351 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 10 Oct 2022 14:31:48 +0200 Subject: [PATCH 26/73] Ensuring dump directory exists on backup step. (#172) * Ensuring dump directory exists on backup step. * Acts on shared storage, so can run once. --- .../database_backup-mysql/tasks/deploy-dump.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index d06b1543..80a6e5ef 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" @@ -14,7 +14,15 @@ when: previous_build_number == 0 run_once: true +- name: Ensure the dump directory exists. + ansible.builtin.file: + path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + run_once: true + - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true From 2c7443277b94486bad5f3d7b57212c9426bf5e9f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 13:49:41 +0200 Subject: [PATCH 27/73] Allowing Drupal 7 jobs to disable cron. (#174) --- roles/cron/cron_drupal7/defaults/main.yml | 1 + roles/cron/cron_drupal7/tasks/cron.yml | 2 +- roles/cron/cron_drupal7/tasks/job.yml | 10 ++++++---- roles/cron/cron_drupal7/tasks/main.yml | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index a09c1036..a6551ab6 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -8,6 +8,7 @@ drupal: - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] diff --git a/roles/cron/cron_drupal7/tasks/cron.yml b/roles/cron/cron_drupal7/tasks/cron.yml index 2bc2c6c7..056d50e0 100644 --- a/roles/cron/cron_drupal7/tasks/cron.yml +++ b/roles/cron/cron_drupal7/tasks/cron.yml @@ -1,6 +1,6 @@ --- - name: Setup Drupal 7 cron tasks. - include_tasks: + ansible.builtin.include_tasks: file: job.yml with_items: "{{ site.cron }}" loop_control: diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index 841c4b51..0f1d3203 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -1,10 +1,10 @@ --- - name: Define cron job command. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\"" when: - drupal.defer is defined @@ -13,7 +13,7 @@ - drupal.defer_target | length > 0 - name: Define cron job command for differing deploy users. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'" when: - www_user != deploy_user @@ -21,9 +21,11 @@ - drupal.defer - name: Setup Drupal cron tasks on app server. - cron: + ansible.builtin.cron: name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" job: "{{ _cron_job_command }}" + state: present + disabled: "{{ entry.disabled | default(omit) }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index 75fa05c7..cf0f80f6 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -10,7 +10,7 @@ - drupal.defer - name: Create site cron entries. - include_tasks: + ansible.builtin.include_tasks: file: cron.yml with_items: "{{ drupal.sites }}" loop_control: From 03852117a42d9aa62e30d3161ae0b418d6e440bf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 17:41:52 +0200 Subject: [PATCH 28/73] Suppress db revert pr 1.x (#177) * Supporting option to suppress reverting backups. * Fixing namespacing in database_backup role. --- .../tasks/cleanup-dump.yml | 2 +- .../tasks/cleanup-rolling.yml | 2 +- .../database_backup-mysql/tasks/cleanup.yml | 12 ++++---- .../tasks/deploy-none.yml | 6 ++-- .../tasks/deploy-rolling.yml | 8 +++--- .../database_backup-mysql/tasks/deploy.yml | 28 +++++++++---------- .../database_backup-mysql/tasks/main.yml | 2 +- .../tasks/revert-dump.yml | 2 +- .../database_backup-mysql/tasks/revert.yml | 14 ++++++---- roles/database_backup/defaults/main.yml | 1 + roles/database_backup/tasks/main.yml | 4 +-- 11 files changed, 42 insertions(+), 39 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index d5774a78..a9ea9ae2 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql dumps. - file: + ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index b131f8e6..2bdf29ee 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql databases. - mysql_db: + community.mysql.mysql_db: name: "{{ database.database }}_{{ item }}" state: absent login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index 71570baa..ac71fcbb 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -1,23 +1,23 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" # We assume it's safe to only go back 50 build back. - name: Delete mysql users. - mysql_user: + community.mysql.mysql_user: name: "{{ database.user }}_{{ item }}" host_all: true state: absent diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml index 65c68978..dea8fe9f 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 1db841b0..71fa7580 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -1,17 +1,17 @@ --- # Setup database names. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}" # Note: we don't use the mysql_db Ansible module on purpose. # If database already exists, we want to fail and not override it # with previous build. - name: Create new database. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index f5a7130a..fc06be74 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -1,27 +1,27 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" # Credentials for the app itself. # Manual: reuse given creds. Note the user gets ignored. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ _mysql_user }}" when: mysql_backup.credentials_handling == 'manual' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ _mysql_password }}" when: mysql_backup.credentials_handling == 'manual' # Static: create/update user/pwd only once. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' @@ -71,18 +71,18 @@ - _legacy_static_creds.stat.exists ### End of legacy handling. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}_{{ build_number }}" when: mysql_backup.credentials_handling == 'rotate' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}" when: mysql_backup.credentials_handling == 'rotate' -- include_tasks: "deploy-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml" # We append privileges instead of replacing, # to allow this role to be looped over, @@ -90,11 +90,11 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true -- set_fact: +- ansible.builtin.set_fact: _build_database: original: "{{ database }}" name: "{{ _mysql_build_database_name }}" @@ -102,5 +102,5 @@ password: "{{ _mysql_build_password }}" host: "{{ _mysql_host }}" -- set_fact: +- ansible.builtin.set_fact: build_databases: "{{ build_databases + [ _build_database ] }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/main.yml b/roles/database_backup/database_backup-mysql/tasks/main.yml index ec10df75..b015ac09 100644 --- a/roles/database_backup/database_backup-mysql/tasks/main.yml +++ b/roles/database_backup/database_backup-mysql/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Take database backup. - include_tasks: "{{ deploy_operation }}.yml" + ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" with_items: "{{ mysql_backup.databases }}" loop_control: loop_var: database diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 735a9805..f914ee78 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,5 @@ --- - name: Revert database from dump. - shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml index ba054b44..f6076262 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml @@ -1,17 +1,19 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "revert-{{ mysql_backup.handling }}.yml" - when: previous_build_number > 0 +- ansible.builtin.include_tasks: "revert-{{ mysql_backup.handling }}.yml" + when: + - previous_build_number > 0 + - database_backup.revert diff --git a/roles/database_backup/defaults/main.yml b/roles/database_backup/defaults/main.yml index 9201fa20..df4881c0 100644 --- a/roles/database_backup/defaults/main.yml +++ b/roles/database_backup/defaults/main.yml @@ -2,3 +2,4 @@ database_backup: engines: - mysql + revert: true \ No newline at end of file diff --git a/roles/database_backup/tasks/main.yml b/roles/database_backup/tasks/main.yml index 83adedf9..829536c2 100644 --- a/roles/database_backup/tasks/main.yml +++ b/roles/database_backup/tasks/main.yml @@ -1,9 +1,9 @@ --- -- set_fact: +- ansible.builtin.set_fact: build_databases: [] - name: Take database dumps. - include_role: + ansible.builtin.include_role: name: "database_backup/database_backup-{{ engine }}" with_items: "{{ database_backup.engines }}" loop_control: From 86f138a15aafe6dd75b2eba1b04ceafb39351782 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 18 Nov 2022 15:05:04 +0100 Subject: [PATCH 29/73] Fixing bad assumption that databases will have TCP connections. (#179) --- .../database_backup/database_backup-mysql/tasks/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index fc06be74..5442230d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -89,11 +89,16 @@ # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. -- name: Create/update mysql user. +- name: Create/update mysql user for TCP connections. ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true +- name: Create/update mysql user for unix socket connections. + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';" + when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + run_once: true + - ansible.builtin.set_fact: _build_database: original: "{{ database }}" From b23654081e8f8a5058d97b2e8b17a785d9a548f2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 16:05:49 +0100 Subject: [PATCH 30/73] Handling the 'drush deploy' command more elegantly for Drupal 8+. (#180) * Handling the 'drush deploy' command more elegantly for Drupal 8+. * Changing logic to AND for cache rebuild. --- .../database_apply/database_apply-drupal8/tasks/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 23cb0b3b..3473d9e8 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -22,7 +22,7 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site - when: previous_build_number == 0 or (site.force_install is defined and site.force_install) + when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. shell: @@ -30,12 +30,14 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site - when: previous_build_number == 0 or (site.force_install is defined and site.force_install) + when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Clear the cache. ansible.builtin.include_role: name: "cache_clear/cache_clear-{{ project_type }}" - when: previous_build_number > 0 + when: + - previous_build_number > 0 + - site.config_import_command != 'deploy' - name: Apply Drupal database updates. command: @@ -46,6 +48,7 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site + when: site.config_import_command != 'deploy' - name: Import configuration. command: From 58069b60c45aef39a655e2d0976b123af5a82fbf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 16:08:26 +0100 Subject: [PATCH 31/73] Attempt to clear the opcache during Drupal deployments. (#182) Co-authored-by: Emlyn Kinzett --- roles/_meta/deploy-drupal7/tasks/main.yml | 2 ++ roles/_meta/deploy-drupal8/tasks/main.yml | 2 ++ roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++ roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++ 4 files changed, 14 insertions(+) diff --git a/roles/_meta/deploy-drupal7/tasks/main.yml b/roles/_meta/deploy-drupal7/tasks/main.yml index 93e6b1b7..cecf3ea0 100644 --- a/roles/_meta/deploy-drupal7/tasks/main.yml +++ b/roles/_meta/deploy-drupal7/tasks/main.yml @@ -24,6 +24,8 @@ name: sanitize/admin_creds - import_role: name: live_symlink +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal7 - import_role: diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 50804566..6ce31d4f 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -26,6 +26,8 @@ name: sanitize/admin_creds - import_role: name: live_symlink +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index eb2edbfa..fe25bcaa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -24,6 +24,11 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 +- name: Clear the opcache. + ansible.builtin.include_role: + name: cache_clear/cache_clear-opcache + when: previous_build_number > 0 + - name: Apply Drupal database updates. shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 3473d9e8..6dd13e22 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -39,6 +39,11 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' +- name: Clear the opcache. + ansible.builtin.include_role: + name: cache_clear/cache_clear-opcache + when: previous_build_number > 0 + - name: Apply Drupal database updates. command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 3ea0bfe4324ea6b9e7acadebba771b4264a764e4 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 18:32:18 +0100 Subject: [PATCH 32/73] Better drush deploy support pr 1.x (#185) * Handling the 'drush deploy' command more elegantly for Drupal 8+. * Changing logic to AND for cache rebuild. * Em's clear cache code needs to run for each 'site' too. --- roles/_init/defaults/main.yml | 2 +- roles/database_apply/database_apply-drupal8/tasks/main.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 790e9409..432485d9 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -10,7 +10,7 @@ drupal: public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 6dd13e22..16f0417b 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -35,6 +35,9 @@ - name: Clear the cache. ansible.builtin.include_role: name: "cache_clear/cache_clear-{{ project_type }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site when: - previous_build_number > 0 - site.config_import_command != 'deploy' From b83534407d373150382dfd431802f5fbf31dc2d4 Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Fri, 30 Dec 2022 13:22:01 +0200 Subject: [PATCH 33/73] Cron job schedule params pr 1.x (#190) * cron job schedule params, namespaces * drupal7-cron-params * cron-job-schedule-params-fix-defaults --- roles/cron/cron_drupal7/defaults/main.yml | 3 +++ roles/cron/cron_drupal7/tasks/job.yml | 3 +++ roles/cron/cron_drupal8/defaults/main.yml | 4 ++++ roles/cron/cron_drupal8/tasks/cron.yml | 2 +- roles/cron/cron_drupal8/tasks/job.yml | 13 +++++++++---- roles/cron/cron_drupal8/tasks/main.yml | 4 ++-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index a6551ab6..ef3ab79f 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -7,6 +7,9 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index 0f1d3203..dd18979c 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -25,6 +25,9 @@ name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" + day: "{{ entry.day | default(omit) }}" + weekday: "{{ entry.weekday | default(omit) }}" + month: "{{ entry.month | default(omit) }}" job: "{{ _cron_job_command }}" state: present disabled: "{{ entry.disabled | default(omit) }}" diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml index 61e2ef8f..913eb530 100644 --- a/roles/cron/cron_drupal8/defaults/main.yml +++ b/roles/cron/cron_drupal8/defaults/main.yml @@ -7,7 +7,11 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] diff --git a/roles/cron/cron_drupal8/tasks/cron.yml b/roles/cron/cron_drupal8/tasks/cron.yml index 2f0c6116..1ed40cc3 100644 --- a/roles/cron/cron_drupal8/tasks/cron.yml +++ b/roles/cron/cron_drupal8/tasks/cron.yml @@ -1,6 +1,6 @@ --- - name: Setup Drupal cron tasks. - include_tasks: + ansible.builtin.include_tasks: file: job.yml with_items: "{{ site.cron }}" loop_control: diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml index bcdac2af..908abff6 100644 --- a/roles/cron/cron_drupal8/tasks/job.yml +++ b/roles/cron/cron_drupal8/tasks/job.yml @@ -1,10 +1,10 @@ --- - name: Define cron job command. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\"" when: - drupal.defer is defined @@ -13,7 +13,7 @@ - drupal.defer_target | length > 0 - name: Define cron job command for differing deploy users. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'" when: - www_user != deploy_user @@ -21,9 +21,14 @@ - drupal.defer - name: Setup Drupal cron tasks on app server. - cron: + ansible.builtin.cron: name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" + day: "{{ entry.day | default(omit) }}" + weekday: "{{ entry.weekday | default(omit) }}" + month: "{{ entry.month | default(omit) }}" job: "{{ _cron_job_command }}" + state: present + disabled: "{{ entry.disabled | default(omit) }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index 75fa05c7..cf0f80f6 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -10,7 +10,7 @@ - drupal.defer - name: Create site cron entries. - include_tasks: + ansible.builtin.include_tasks: file: cron.yml with_items: "{{ drupal.sites }}" loop_control: From e97e852235ff29185e7d52ceffe9c28f8e87a11f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 23 Jan 2023 17:21:16 +0100 Subject: [PATCH 34/73] Adding option to stop services that might interfere with a squashfs mount. (#193) --- roles/deploy_code/defaults/main.yml | 5 ++++- roles/deploy_code/tasks/cleanup.yml | 33 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index f1f7cdb4..f9b84739 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -25,11 +25,14 @@ deploy_code: # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" - # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount. + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. services: [] # services: # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false # Details of API call to trigger. See api_call role. diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 353854b3..a5da0d87 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -144,6 +144,23 @@ loop_var: www_service become: true when: + - deploy_code.service_action == "reload" + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - deploy_code.services | length > 0 + +- name: Stop any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: stopped + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" @@ -169,6 +186,22 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" +- name: Start any services we stopped. + ansible.builtin.service: + name: "{{ www_service }}" + state: started + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - deploy_code.services | length > 0 + - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: name: api_call From 7bbb77fbd07bedcbc9d19f0d5354b03b1b34186b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 25 Jan 2023 15:32:19 +0100 Subject: [PATCH 35/73] Drush refactor pr 1.x (#197) * Updating docs. * Consistent drush handling in cron. * Linting cruft and removing unimplemented drush.yml handling from D7. * Refactor drush role to support new GitHub release format. * Updating drush README. * Casting the first part of the drush version string as an integer. * Refactoring integer handling. * Major version string will always be a string, assuming below 8 will never exist. * Downloaded drush archive is remote, not local. * Refactoring handling of .bin directory. * Supporting drush-launcher. * Removing drush installation support for Drupal 8 and above. * Bad variable name in Drupal cron jobs. * Fixing maintenance mode command linting and consistency. * Moving chdir to args to see if it helps. * Trying an actual 'cd' command inline. * Cannot '&&' in a cmd. * Trying drush's --root flag to set Drupal path. * With --root we do not need chdir for drush. * Revert "With --root we do not need chdir for drush." This reverts commit d07aaf798a8f5699ed669f9411db2810e2bcd9d8. --- docs/_Sidebar.md | 2 + docs/roles/_init.md | 6 ++- docs/roles/_meta/deploy-simplesamlphp.md | 9 +++++ docs/roles/api_call.md | 26 +++++++++++++ docs/roles/cli/cachetool.md | 2 +- docs/roles/cli/drush.md | 10 +++-- .../cron_database_backup-mysql.md | 1 + docs/roles/cron/cron_drupal7.md | 6 ++- docs/roles/cron/cron_drupal8.md | 7 +++- docs/roles/database_backup.md | 2 +- .../database_backup/database_backup-mysql.md | 1 + docs/roles/deploy_code.md | 37 ++++++++++++++++++- .../sync/database_sync/database_sync-mysql.md | 10 +++++ roles/_init/README.md | 6 ++- roles/_init/defaults/main.yml | 1 + roles/_init/tasks/drupal7.yml | 2 +- roles/_init/tasks/drupal8.yml | 6 +-- roles/api_call/README.md | 17 +++++++++ roles/cli/cachetool/README.md | 2 +- roles/cli/drush/README.md | 10 +++-- roles/cli/drush/defaults/main.yml | 5 +-- roles/cli/drush/tasks/main.yml | 11 +++--- .../config_generate-drupal7/tasks/drush.yml | 10 ----- .../config_generate-drupal7/tasks/main.yml | 10 +---- .../config_generate-drupal8/tasks/drush.yml | 3 +- .../config_generate-drupal8/tasks/main.yml | 4 +- .../config_generate-matomo/tasks/main.yml | 3 +- .../config_generate-mautic/tasks/main.yml | 11 +++--- roles/config_generate/tasks/main.yml | 7 ++-- .../cron_database_backup-mysql/README.md | 1 + roles/cron/cron_drupal7/README.md | 6 ++- roles/cron/cron_drupal7/defaults/main.yml | 1 + roles/cron/cron_drupal7/tasks/job.yml | 2 +- roles/cron/cron_drupal8/README.md | 7 +++- roles/cron/cron_drupal8/defaults/main.yml | 3 +- roles/cron/cron_drupal8/tasks/job.yml | 2 +- roles/database_backup/README.md | 2 +- .../database_backup-mysql/README.md | 1 + roles/deploy_code/README.md | 37 ++++++++++++++++++- .../tasks/main.yml | 2 +- .../tasks/offline.yml | 11 +++--- .../tasks/online.yml | 10 +++-- .../database_sync-mysql/README.md | 10 +++++ 43 files changed, 233 insertions(+), 89 deletions(-) create mode 100644 docs/roles/_meta/deploy-simplesamlphp.md create mode 100644 docs/roles/api_call.md delete mode 100644 roles/config_generate/config_generate-drupal7/tasks/drush.yml diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index efa27c64..16139d8f 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -3,6 +3,7 @@ - [Install](/install) - [Usage](/scripts) - [Roles](roles) + - [API call](/roles/api_call) - [Config](/roles/cache_clear) - [Drupal 7](/roles/cache_clear/cache_clear-drupal7) - [Drupal 8](/roles/cache_clear/cache_clear-drupal8) @@ -37,6 +38,7 @@ - [Drupal 8](/roles/_meta/deploy-drupal8) - [Matomo](/roles/_meta/deploy-matomo) - [Mautic](/roles/_meta/deploy-mautic) + - [SimpleSAMLphp](/roles/_meta/deploy-simplesamlphp) - [NPM](/roles/npm) - [Sync roles](/roles/sync) - [Database sync](/roles/sync/database_sync) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index f733d1a3..1c7e835e 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least. --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" @@ -32,7 +34,7 @@ drupal: mautic: image_path: "media/images" force_install: false - +bin_directory: "/home/{{ deploy_user }}/.bin" ``` diff --git a/docs/roles/_meta/deploy-simplesamlphp.md b/docs/roles/_meta/deploy-simplesamlphp.md new file mode 100644 index 00000000..3d82e671 --- /dev/null +++ b/docs/roles/_meta/deploy-simplesamlphp.md @@ -0,0 +1,9 @@ +# SimpleSAMLphp +Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer. + +This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision: + +* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx + + + diff --git a/docs/roles/api_call.md b/docs/roles/api_call.md new file mode 100644 index 00000000..15ea2788 --- /dev/null +++ b/docs/roles/api_call.md @@ -0,0 +1,26 @@ +# API call + +Making RESTful API calls to other platforms. + + + + + +## Default variables +```yaml +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 +``` + + diff --git a/docs/roles/cli/cachetool.md b/docs/roles/cli/cachetool.md index 833eb8fc..9cba4a32 100644 --- a/docs/roles/cli/cachetool.md +++ b/docs/roles/cli/cachetool.md @@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: 8.2.2 + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version ``` diff --git a/docs/roles/cli/drush.md b/docs/roles/cli/drush.md index 30f02549..41b673c2 100644 --- a/docs/roles/cli/drush.md +++ b/docs/roles/cli/drush.md @@ -1,13 +1,15 @@ # Drush -Installs the `drush` command-line tool for the deploy user. +Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23). + +For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush` documentation](https://www.drush.org/latest/install/). + ## Default variables ```yaml --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false ``` diff --git a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md index ee71e2bf..4d5bdab5 100644 --- a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md +++ b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md @@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases. cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here ``` diff --git a/docs/roles/cron/cron_drupal7.md b/docs/roles/cron/cron_drupal7.md index 8222ddaf..14356d15 100644 --- a/docs/roles/cron/cron_drupal7.md +++ b/docs/roles/cron/cron_drupal7.md @@ -14,12 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/docs/roles/cron/cron_drupal8.md b/docs/roles/cron/cron_drupal8.md index ae25adc4..bafd49b0 100644 --- a/docs/roles/cron/cron_drupal8.md +++ b/docs/roles/cron/cron_drupal8.md @@ -14,13 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/docs/roles/database_backup.md b/docs/roles/database_backup.md index 947c8053..772efa70 100644 --- a/docs/roles/database_backup.md +++ b/docs/roles/database_backup.md @@ -7,7 +7,7 @@ Generate backups for each build. database_backup: engines: - mysql - + revert: true ``` diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md index 5b0d7289..b52413bf 100644 --- a/docs/roles/database_backup/database_backup-mysql.md +++ b/docs/roles/database_backup/database_backup-mysql.md @@ -7,6 +7,7 @@ Generate MySQL backups for each build. mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md index bad0040f..4e371f28 100644 --- a/docs/roles/deploy_code.md +++ b/docs/roles/deploy_code.md @@ -11,7 +11,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -23,12 +23,45 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 ``` diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index bcb9c6ad..cf7e5311 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -5,6 +5,7 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. @@ -20,6 +21,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -30,6 +38,8 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index f733d1a3..1c7e835e 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least. --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" @@ -32,7 +34,7 @@ drupal: mautic: image_path: "media/images" force_install: false - +bin_directory: "/home/{{ deploy_user }}/.bin" ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 432485d9..b38c1e59 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -25,3 +25,4 @@ drupal: mautic: image_path: "media/images" force_install: false +bin_directory: "/home/{{ deploy_user }}/.bin" \ No newline at end of file diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml index 6487564c..133b3d73 100644 --- a/roles/_init/tasks/drupal7.yml +++ b/roles/_init/tasks/drupal7.yml @@ -1,7 +1,7 @@ --- - name: Define Drush path. set_fact: - drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}" + drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}" - name: Ensure we have Drush binary. import_role: diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml index 6487564c..ebe63b82 100644 --- a/roles/_init/tasks/drupal8.yml +++ b/roles/_init/tasks/drupal8.yml @@ -1,8 +1,4 @@ --- - name: Define Drush path. set_fact: - drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}" - -- name: Ensure we have Drush binary. - import_role: - name: cli/drush + drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}" diff --git a/roles/api_call/README.md b/roles/api_call/README.md index 83d1ab28..15ea2788 100644 --- a/roles/api_call/README.md +++ b/roles/api_call/README.md @@ -6,4 +6,21 @@ Making RESTful API calls to other platforms. +## Default variables +```yaml +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 +``` + diff --git a/roles/cli/cachetool/README.md b/roles/cli/cachetool/README.md index 833eb8fc..9cba4a32 100644 --- a/roles/cli/cachetool/README.md +++ b/roles/cli/cachetool/README.md @@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: 8.2.2 + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version ``` diff --git a/roles/cli/drush/README.md b/roles/cli/drush/README.md index 30f02549..41b673c2 100644 --- a/roles/cli/drush/README.md +++ b/roles/cli/drush/README.md @@ -1,13 +1,15 @@ # Drush -Installs the `drush` command-line tool for the deploy user. +Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23). + +For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush` documentation](https://www.drush.org/latest/install/). + ## Default variables ```yaml --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false ``` diff --git a/roles/cli/drush/defaults/main.yml b/roles/cli/drush/defaults/main.yml index 68ff2c66..746fce44 100644 --- a/roles/cli/drush/defaults/main.yml +++ b/roles/cli/drush/defaults/main.yml @@ -1,6 +1,5 @@ --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false \ No newline at end of file diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml index 4d27aa3e..5c73be51 100644 --- a/roles/cli/drush/tasks/main.yml +++ b/roles/cli/drush/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Check if we already have a global Drush. - stat: + ansible.builtin.stat: path: "{{ drush_bin }}" register: drush_global when: @@ -16,18 +16,19 @@ register: drush_global_directory - name: Check if installed Drush version match. - shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" register: drush_global_version when: - deploy_operation == 'deploy' - drush_global.stat.exists - not drush.use_vendor -- name: Download Drush installer. - get_url: +- name: Download Drush installer for Drupal 7. + ansible.builtin.get_url: url: "https://github.com/drush-ops/drush/releases/download/{{ drush.version }}/drush.phar" dest: "{{ drush_bin }}" mode: 0755 + force: true when: - deploy_operation == 'deploy' - not drush.use_vendor @@ -49,7 +50,7 @@ - deploy_operation == 'deploy' - name: Trigger Drush init tasks. - command: "{{ drush_bin }} init -y" + ansible.builtin.command: "{{ drush_bin }} init -y" when: - deploy_operation == 'deploy' - not drush.use_vendor diff --git a/roles/config_generate/config_generate-drupal7/tasks/drush.yml b/roles/config_generate/config_generate-drupal7/tasks/drush.yml deleted file mode 100644 index 9c85eb9d..00000000 --- a/roles/config_generate/config_generate-drupal7/tasks/drush.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -- name: Generates drush.yml file. - template: - src: '{{ item }}' - dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml" - with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml.j2" - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml" - - "drush.yml.j2" \ No newline at end of file diff --git a/roles/config_generate/config_generate-drupal7/tasks/main.yml b/roles/config_generate/config_generate-drupal7/tasks/main.yml index b984fd58..23781ded 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/main.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/main.yml @@ -1,14 +1,6 @@ --- - name: Generates settings file. - include_tasks: settings.yml + ansible.builtin.include_tasks: settings.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site - -#- name: Generates Drush settings file. -# include_tasks: drush.yml -# when: -# - site.base_url is defined -# with_items: "{{ drupal.sites }}" -# loop_control: -# loop_var: site diff --git a/roles/config_generate/config_generate-drupal8/tasks/drush.yml b/roles/config_generate/config_generate-drupal8/tasks/drush.yml index 9c85eb9d..f4b7ae7a 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/drush.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/drush.yml @@ -1,7 +1,6 @@ --- - - name: Generates drush.yml file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml" with_first_found: diff --git a/roles/config_generate/config_generate-drupal8/tasks/main.yml b/roles/config_generate/config_generate-drupal8/tasks/main.yml index cd3c19fd..3db5bf60 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/main.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/main.yml @@ -1,12 +1,12 @@ --- - name: Generates settings file. - include_tasks: settings.yml + ansible.builtin.include_tasks: settings.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site - name: Generates Drush settings file. - include_tasks: drush.yml + ansible.builtin.include_tasks: drush.yml when: - site.base_url is defined with_items: "{{ drupal.sites }}" diff --git a/roles/config_generate/config_generate-matomo/tasks/main.yml b/roles/config_generate/config_generate-matomo/tasks/main.yml index 505ff3ce..e5482e65 100644 --- a/roles/config_generate/config_generate-matomo/tasks/main.yml +++ b/roles/config_generate/config_generate-matomo/tasks/main.yml @@ -1,7 +1,6 @@ --- - - name: Generates config file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/config/config.ini.php" with_first_found: diff --git a/roles/config_generate/config_generate-mautic/tasks/main.yml b/roles/config_generate/config_generate-mautic/tasks/main.yml index d1da20c5..2a9d63d0 100644 --- a/roles/config_generate/config_generate-mautic/tasks/main.yml +++ b/roles/config_generate/config_generate-mautic/tasks/main.yml @@ -1,15 +1,14 @@ --- - -- name: set public files path. - set_fact: +- name: Set public files path. + ansible.builtin.set_fact: build_image_path: "{{ mautic.image_path }}" -- name: set private files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_upload_dir: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_private_files" - name: Generates local.php file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/app/config/local.php" with_first_found: diff --git a/roles/config_generate/tasks/main.yml b/roles/config_generate/tasks/main.yml index 3bce314b..2af08e87 100644 --- a/roles/config_generate/tasks/main.yml +++ b/roles/config_generate/tasks/main.yml @@ -1,12 +1,11 @@ --- - - name: Generate project specific configuration. - include_role: + ansible.builtin.include_role: name: "config_generate/config_generate-{{ project_type }}" when: deploy_operation == 'deploy' - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ config_generate.templates }}" @@ -17,7 +16,7 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md index ee71e2bf..4d5bdab5 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md @@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases. cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here ``` diff --git a/roles/cron/cron_drupal7/README.md b/roles/cron/cron_drupal7/README.md index 8222ddaf..14356d15 100644 --- a/roles/cron/cron_drupal7/README.md +++ b/roles/cron/cron_drupal7/README.md @@ -14,12 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index ef3ab79f..419e9c4b 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -16,3 +16,4 @@ drupal: defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" \ No newline at end of file diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index dd18979c..b89a0b05 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -1,7 +1,7 @@ --- - name: Define cron job command. ansible.builtin.set_fact: - _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}" + _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). ansible.builtin.set_fact: diff --git a/roles/cron/cron_drupal8/README.md b/roles/cron/cron_drupal8/README.md index ae25adc4..bafd49b0 100644 --- a/roles/cron/cron_drupal8/README.md +++ b/roles/cron/cron_drupal8/README.md @@ -14,13 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml index 913eb530..9d4b90f6 100644 --- a/roles/cron/cron_drupal8/defaults/main.yml +++ b/roles/cron/cron_drupal8/defaults/main.yml @@ -16,5 +16,4 @@ drupal: defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml index 908abff6..b89a0b05 100644 --- a/roles/cron/cron_drupal8/tasks/job.yml +++ b/roles/cron/cron_drupal8/tasks/job.yml @@ -1,7 +1,7 @@ --- - name: Define cron job command. ansible.builtin.set_fact: - _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}" + _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). ansible.builtin.set_fact: diff --git a/roles/database_backup/README.md b/roles/database_backup/README.md index 947c8053..772efa70 100644 --- a/roles/database_backup/README.md +++ b/roles/database_backup/README.md @@ -7,7 +7,7 @@ Generate backups for each build. database_backup: engines: - mysql - + revert: true ``` diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md index 5b0d7289..b52413bf 100644 --- a/roles/database_backup/database_backup-mysql/README.md +++ b/roles/database_backup/database_backup-mysql/README.md @@ -7,6 +7,7 @@ Generate MySQL backups for each build. mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md index bad0040f..4e371f28 100644 --- a/roles/deploy_code/README.md +++ b/roles/deploy_code/README.md @@ -11,7 +11,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -23,12 +23,45 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 ``` diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml index 0633bc75..ccc2c2cf 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml @@ -1,5 +1,5 @@ --- -- include_tasks: "{{ maintenance_mode.operation }}.yml" +- ansible.builtin.include_tasks: "{{ maintenance_mode.operation }}.yml" with_items: "{{ drupal.sites }}" loop_control: loop_var: site diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index cbe65b07..5fd4ddd0 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -1,8 +1,8 @@ --- -# @todo this needs refactoring for multisite. - name: Enable maintenance mode. - command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -11,8 +11,9 @@ - project_type == 'drupal8' - name: Enable maintenance mode D7. - shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index e6a9cbf5..8337c968 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,7 +1,8 @@ --- - name: Disable maintenance mode. - command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -9,8 +10,9 @@ - project_type == 'drupal8' - name: Disable maintenance mode D7. - shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index bcb9c6ad..cf7e5311 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -5,6 +5,7 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. @@ -20,6 +21,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -30,6 +38,8 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" ``` From cc31f6a261b71e913ee706d647958ad1b545715e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 27 Jan 2023 14:28:15 +0100 Subject: [PATCH 36/73] Better deploy_code role docs. (#213) * Better deploy_code role docs. * roles path error in docs. * roles path error in docs. * Adding a note about deploy_previous handling for squashfs. --- docs/roles/deploy_code.md | 80 ++++++++++++++++++++++++++++++++++++- roles/deploy_code/README.md | 80 ++++++++++++++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 2 deletions(-) diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md index 4e371f28..30d87b11 100644 --- a/docs/roles/deploy_code.md +++ b/docs/roles/deploy_code.md @@ -1,5 +1,83 @@ # Deploy -Step that deploys the codebase. +Step that deploys the codebase. On standalone machines and "static" clusters of web servers (e.g. machines whose addressing never changes) this is reasonably straightforward, the default variables should "just work". This role also supports deployment to autoscaling clusters of web servers, such as AWS autoscaling groups or containerised architecture. More details on that after this section. + +The shell script that wraps Ansible to handle the various build steps has various "stages" and the `deploy_code` role has a set of tasks for each stage. The key one for code building on the static/current cluster servers is [the `deploy.yml` file](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/deploy.yml). Here you will find the steps for checking out and building code on web servers, as well as the loading in of any application specific deploy code, [e.g. these tasks for Drupal 8](https://github.com/codeenigma/ce-deploy/tree/1.x/roles/deploy_code/deploy_code-drupal8/tasks). You choose what extra tasks to load via the `project_type` variable. Current core options are: + +* `drupal7` +* `drupal8` +* `matomo` +* `mautic` +* `simplesamlphp` + +Patches to support other common applications are always welcome! Also, Ansible inheritance being what it is, you can create your own custom deploy role in the same directory as your deployment playbook and Ansible will detect it and make it available to you. For example, if you create `./roles/deploy_code/deploy_code-myapp/tasks/main.yml` relative to your playbook and set `project_type: myapp` in your project variables then `ce-deploy` will load in those tasks. + +# Autoscale deployment +For autoscaling clusters - no matter the underlying tech - the build code needs to be stored somewhere central and accessible to any potential new servers in the cluster. Because the performance of network attached storage (NAS) is often too poor or unreliable, we do not deploy the code to NAS - although this would be the simplest approach. Instead the technique we use is to build the code on each current server in the cluster, as though it were a static cluster or standalone machine, but *also* copy the code to the NAS so it is available to all future machines. This makes the existence of mounted NAS that is attached to all new servers a pre-requisite for `ce-deploy` to work with autoscaling. + +**Important**, autoscale deployments need to be carefully co-ordinated with [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) so new servers/containers have the correct scripts in place to place their code after they initialise. Specifically, the `mount_sync.tarballs` or `mount_sync.squashed_fs` list variables in `ce-provision` must contain paths that match with the location specified in the `deploy_code.mount_sync` variable in `ce-deploy` so `ce-deploy` copies code to the place `ce-provision`'s `cloud-init` scripts expect to find it. (More on the use of `cloud-init` below.) + +(An aside, we have previously supported S3-like object storage for storing the built code, but given all the applications we work with need to have NAS anyway for end user file uploads and other shared cluster resources, it seems pointless to introduce a second storage mechanism when we have one there already that works just fine.) + +This packaging of a copy of the code all happens in [the `cleanup.yml` file of the role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml). It supports three options: + +* No autoscale (or AWS AMI-based autoscale - see below) - leave `mount_sync` as an empty string +* `tarball` type - makes a `tar.gz` with the code in and copies it to the NAS +* `squashfs` type - packs a [`squashfs`](https://github.com/plougher/squashfs-tools) image, copies to the NAS and mounts it on each web server + +For both `tarball` and `squashfs` you need to set `mount_type` accordingly and the `mount_sync` variable to the location on your NAS where you want to store the built code. + +## `tarball` builds +This is the simplest method of autoscale deployment, it simply packs up the code and copies it to the NAS at the end of the deployment. Everything else is just a standard "normal" build. + +**Important**, this method is only appropriate if you do not have too many files to deploy. The packing and restoring takes a very long time if there are many small files, so it is not appropriate for things like `composer` built PHP applications. + +### Rolling back +With this method the live code directory is also the build directory, therefore you can edit the code in place in an emergency and "rolling back" if there are issues with a build is just a case of pointing the live build symlink back to the previous build. As long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the application. If the backup is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. By default this will be on the NAS so it is available to all web servers. + +## `squashfs` builds +Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command). + +However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) + +Consequently, at the build stage there are two important extra variables to set: + +```yaml +deploy_code: + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload +``` + +`services` is a list of Linux services to stop/reload in order to ensure the mount point is not locked. Usually this will be your PHP service, e.g. + +```yaml +deploy_code: + services: + - php8.1-fpm +``` + +`service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time. + +Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory. + +### Rolling back +Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event. + +Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. + +Emergency code changes are possible but more fiddly. You have to copy the codebase from the mount to a sensible, *writeable* location, make your changes, [use the `squashfs` command to pack a new image](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L54), mount that image and, crucially, replace the `deploy.sqsh` image file on the NAS with your new image so future autoscale events will pick it up. + +# Autoscaling events +Deploying code with autoscaling clusters relies on [cloud-init](https://cloudinit.readthedocs.io/) and is managed in our stack by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync). Whenever a new server spins up in a cluster, the `cloud-init` run-once script put in place by `ce-provision` is executed and that copies down the code from the NAS and deploys it to the correct location on the new server. At that point the server should become "healthy" and start serving the application. + +# AMI-based autoscale +**This is experimental.** We are heavily based on [GitLab CE](https://gitlab.com/rluna-gitlab/gitlab-ce) and one of the options we support with [our provisioning tools](https://github.com/codeenigma/ce-provision/tree/1.x) is packing an [AWS AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the code embedded within, thus no longer requiring the [cloud-init](https://cloudinit.readthedocs.io/) step at all. [We call this option `repack` and the code is here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/aws/aws_ami/tasks/repack.yml) This makes provisioning of new machines in a cluster a little faster than the `squashfs` option, but requires the ability to trigger a build on our infrastructure `controller` server to execute a cluster build and pack the AMI. That is what the `api_call` dictionary below is providing for. You can see the API call constructed in [the last task of `cleanup.yml`](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L205). + diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md index 4e371f28..30d87b11 100644 --- a/roles/deploy_code/README.md +++ b/roles/deploy_code/README.md @@ -1,5 +1,83 @@ # Deploy -Step that deploys the codebase. +Step that deploys the codebase. On standalone machines and "static" clusters of web servers (e.g. machines whose addressing never changes) this is reasonably straightforward, the default variables should "just work". This role also supports deployment to autoscaling clusters of web servers, such as AWS autoscaling groups or containerised architecture. More details on that after this section. + +The shell script that wraps Ansible to handle the various build steps has various "stages" and the `deploy_code` role has a set of tasks for each stage. The key one for code building on the static/current cluster servers is [the `deploy.yml` file](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/deploy.yml). Here you will find the steps for checking out and building code on web servers, as well as the loading in of any application specific deploy code, [e.g. these tasks for Drupal 8](https://github.com/codeenigma/ce-deploy/tree/1.x/roles/deploy_code/deploy_code-drupal8/tasks). You choose what extra tasks to load via the `project_type` variable. Current core options are: + +* `drupal7` +* `drupal8` +* `matomo` +* `mautic` +* `simplesamlphp` + +Patches to support other common applications are always welcome! Also, Ansible inheritance being what it is, you can create your own custom deploy role in the same directory as your deployment playbook and Ansible will detect it and make it available to you. For example, if you create `./roles/deploy_code/deploy_code-myapp/tasks/main.yml` relative to your playbook and set `project_type: myapp` in your project variables then `ce-deploy` will load in those tasks. + +# Autoscale deployment +For autoscaling clusters - no matter the underlying tech - the build code needs to be stored somewhere central and accessible to any potential new servers in the cluster. Because the performance of network attached storage (NAS) is often too poor or unreliable, we do not deploy the code to NAS - although this would be the simplest approach. Instead the technique we use is to build the code on each current server in the cluster, as though it were a static cluster or standalone machine, but *also* copy the code to the NAS so it is available to all future machines. This makes the existence of mounted NAS that is attached to all new servers a pre-requisite for `ce-deploy` to work with autoscaling. + +**Important**, autoscale deployments need to be carefully co-ordinated with [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) so new servers/containers have the correct scripts in place to place their code after they initialise. Specifically, the `mount_sync.tarballs` or `mount_sync.squashed_fs` list variables in `ce-provision` must contain paths that match with the location specified in the `deploy_code.mount_sync` variable in `ce-deploy` so `ce-deploy` copies code to the place `ce-provision`'s `cloud-init` scripts expect to find it. (More on the use of `cloud-init` below.) + +(An aside, we have previously supported S3-like object storage for storing the built code, but given all the applications we work with need to have NAS anyway for end user file uploads and other shared cluster resources, it seems pointless to introduce a second storage mechanism when we have one there already that works just fine.) + +This packaging of a copy of the code all happens in [the `cleanup.yml` file of the role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml). It supports three options: + +* No autoscale (or AWS AMI-based autoscale - see below) - leave `mount_sync` as an empty string +* `tarball` type - makes a `tar.gz` with the code in and copies it to the NAS +* `squashfs` type - packs a [`squashfs`](https://github.com/plougher/squashfs-tools) image, copies to the NAS and mounts it on each web server + +For both `tarball` and `squashfs` you need to set `mount_type` accordingly and the `mount_sync` variable to the location on your NAS where you want to store the built code. + +## `tarball` builds +This is the simplest method of autoscale deployment, it simply packs up the code and copies it to the NAS at the end of the deployment. Everything else is just a standard "normal" build. + +**Important**, this method is only appropriate if you do not have too many files to deploy. The packing and restoring takes a very long time if there are many small files, so it is not appropriate for things like `composer` built PHP applications. + +### Rolling back +With this method the live code directory is also the build directory, therefore you can edit the code in place in an emergency and "rolling back" if there are issues with a build is just a case of pointing the live build symlink back to the previous build. As long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the application. If the backup is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. By default this will be on the NAS so it is available to all web servers. + +## `squashfs` builds +Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command). + +However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) + +Consequently, at the build stage there are two important extra variables to set: + +```yaml +deploy_code: + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload +``` + +`services` is a list of Linux services to stop/reload in order to ensure the mount point is not locked. Usually this will be your PHP service, e.g. + +```yaml +deploy_code: + services: + - php8.1-fpm +``` + +`service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time. + +Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory. + +### Rolling back +Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event. + +Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. + +Emergency code changes are possible but more fiddly. You have to copy the codebase from the mount to a sensible, *writeable* location, make your changes, [use the `squashfs` command to pack a new image](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L54), mount that image and, crucially, replace the `deploy.sqsh` image file on the NAS with your new image so future autoscale events will pick it up. + +# Autoscaling events +Deploying code with autoscaling clusters relies on [cloud-init](https://cloudinit.readthedocs.io/) and is managed in our stack by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync). Whenever a new server spins up in a cluster, the `cloud-init` run-once script put in place by `ce-provision` is executed and that copies down the code from the NAS and deploys it to the correct location on the new server. At that point the server should become "healthy" and start serving the application. + +# AMI-based autoscale +**This is experimental.** We are heavily based on [GitLab CE](https://gitlab.com/rluna-gitlab/gitlab-ce) and one of the options we support with [our provisioning tools](https://github.com/codeenigma/ce-provision/tree/1.x) is packing an [AWS AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the code embedded within, thus no longer requiring the [cloud-init](https://cloudinit.readthedocs.io/) step at all. [We call this option `repack` and the code is here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/aws/aws_ami/tasks/repack.yml) This makes provisioning of new machines in a cluster a little faster than the `squashfs` option, but requires the ability to trigger a build on our infrastructure `controller` server to execute a cluster build and pack the AMI. That is what the `api_call` dictionary below is providing for. You can see the API call constructed in [the last task of `cleanup.yml`](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L205). + From f6a3cb362eae9bb05d9d2dee3f8cd41f7ddcebae Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Fri, 27 Jan 2023 19:42:04 +0200 Subject: [PATCH 37/73] Cachetool settings and drupal7 tasks fix pr 1.x (#211) * cachetool_settings_and_drupal7_tasks_fix * cachetool_settings_and_drupal7_tasks_fix * cachetool_settings_and_drupal7_tasks_fix * cachetool settings fixing condition and namespace * fixing cachetool adapter options * fixed var * fixed var * fixed var --- .../cache_clear-opcache/defaults/main.yml | 2 +- .../cache_clear-opcache/tasks/main.yml | 16 +++-- roles/cli/cachetool/defaults/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 68 ++++++++++++++++--- .../tasks/offline.yml | 2 +- .../tasks/online.yml | 2 +- 6 files changed, 76 insertions(+), 16 deletions(-) diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 8121226e..97623e62 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -4,7 +4,7 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - adapter: "" + # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index bee9e6fa..57feac51 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -1,16 +1,24 @@ --- +- name: Get latest php installed + ansible.builtin.shell: + cmd: "ls /etc/php/ | tail -1" + register: _php_version + +- name: Set cachetool adapter. + ansible.builtin.set_fact: + _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('--fcgi=127.0.0.1:90' + _php_version.stdout | replace('.','')) }}" - name: Clear opcache. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n opcache:reset" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset" when: cache_clear_opcache.clear_opcache - name: Clear apcu. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n apcu:cache:clear all" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear all" when: cache_clear_opcache.clear_apcu - name: Clear stats. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n stat:clear" - when: cache_clear_opcache.clear_stat \ No newline at end of file + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear" + when: cache_clear_opcache.clear_stat diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index 5ec1b2d7..8b98934d 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version \ No newline at end of file + version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index aa27df04..12cba942 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -13,22 +13,74 @@ when: - deploy_operation == 'deploy' -- name: Download latest cachetool installer. - get_url: - url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" - dest: "{{ cachetool_bin }}" - mode: 0755 +- name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility + block: + - name: Get latest php installed + ansible.builtin.shell: + cmd: "ls /etc/php/ | tail -1 | sed -e 's/\\.//g'" + register: _php_version_str + + - name: Download latest cachetool installer if PHP is 8.1 or newer. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int >= 81 + + - name: Download cachetool version 8.5.0 installer if PHP is 8.0. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 80 + + - name: Download cachetool version 7.1.0 installer if PHP is 7.3 or newer. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int >= 73 + - _php_version_str.stdout | int < 80 + + - name: Download cachetool version 5.1.3 installer if PHP is 7.2. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 72 + + - name: Download cachetool version 4.1.1 installer if PHP is 7.1. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 71 + + - name: Download cachetool version 3.2.2 installer if PHP version is too old. + ansible.builtin.get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int < 71 + when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - - cachetool.version == 'latest' + - cachetool.version | length == 0 - name: "Download cachetool version {{ cachetool.version }} installer." - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" dest: "{{ cachetool_bin }}" mode: 0755 when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - - cachetool.version != 'latest' + - cachetool.version is defined + - cachetool.version | length > 0 diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 5fd4ddd0..f8ae276c 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -12,7 +12,7 @@ - name: Enable maintenance mode D7. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 8337c968..dfc3e3b3 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -11,7 +11,7 @@ - name: Disable maintenance mode D7. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" From 44fef49398aec66a187d7c05c00d53ba9023cf6b Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Tue, 31 Jan 2023 13:02:14 +0200 Subject: [PATCH 38/73] previous_cachetool_remove_before_install_new (#224) * previous_cachetool_remove_before_install_new * namespace fix --- roles/cli/cachetool/tasks/main.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 12cba942..2e042957 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -1,17 +1,8 @@ --- -- name: Check if we already have cachetool. - stat: +- name: Remove previous cachetool if exists. + ansible.builtin.file: path: "{{ cachetool_bin }}" - register: cachetool_global - when: - - deploy_operation == 'deploy' - -- name: Ensure bin directory exists. - file: - path: "{{ cachetool_bin | dirname }}" - state: directory - when: - - deploy_operation == 'deploy' + state: absent - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility block: @@ -71,7 +62,6 @@ when: - deploy_operation == 'deploy' - - not cachetool_global.stat.exists - cachetool.version | length == 0 - name: "Download cachetool version {{ cachetool.version }} installer." @@ -81,6 +71,5 @@ mode: 0755 when: - deploy_operation == 'deploy' - - not cachetool_global.stat.exists - cachetool.version is defined - cachetool.version | length > 0 From f1b2b902c80981fed27458a7640a6ddbfdae15ba Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Tue, 31 Jan 2023 14:24:16 +0200 Subject: [PATCH 39/73] Fix cachetool removal pr 1.x (#227) * fix_cachetool_removal_step * fix task name --- roles/cli/cachetool/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 2e042957..499b9ebd 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -3,6 +3,8 @@ ansible.builtin.file: path: "{{ cachetool_bin }}" state: absent + when: + - deploy_operation == 'deploy' - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility block: @@ -64,7 +66,7 @@ - deploy_operation == 'deploy' - cachetool.version | length == 0 -- name: "Download cachetool version {{ cachetool.version }} installer." +- name: "Download the specified {{ cachetool.version }} cachetool version installer." ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" dest: "{{ cachetool_bin }}" From de6d80bdb30ef46ec5dd716a0781b35b1432c5cd Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:29:08 +0200 Subject: [PATCH 40/73] Fix cachetool removal pr 1.x (#229) * fix_cachetool_removal_step * fix task name * fix_cachetool_removal_task * fix_cachetool_removal_task --- roles/cli/cachetool/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 499b9ebd..f0f05b28 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -6,6 +6,13 @@ when: - deploy_operation == 'deploy' +- name: Ensure bin directory exists. + ansible.builtin.file: + path: "{{ cachetool_bin | dirname }}" + state: directory + when: + - deploy_operation == 'deploy' + - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility block: - name: Get latest php installed From 1eda678122c41d5e3da08a03f8e70f6fd0f865ba Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 3 Feb 2023 12:45:36 +0100 Subject: [PATCH 41/73] Documentation enhancements pr 1.x (#218) * Better deploy_code role docs. * roles path error in docs. * roles path error in docs. * Adding a note about deploy_previous handling for squashfs. * Reference incorrect role for deploy user sudo perms. * Minor edits to frontpage README. * Rebuilt docs. * Accidentally overwrote docs change. --- README.md | 5 ++--- docs/roles/cache_clear/cache_clear-opcache.md | 2 +- docs/roles/cli/cachetool.md | 3 ++- docs/roles/deploy_code.md | 6 +++--- roles/cache_clear/cache_clear-opcache/README.md | 2 +- roles/cli/cachetool/README.md | 3 ++- roles/deploy_code/README.md | 6 +++--- .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 13 ++++++++----- .../admin_creds/admin_creds-drupal8/tasks/admin.yml | 6 +++--- 9 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 41910905..3046acb1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,10 @@ # ce-deploy -[![Build Status](https://api.travis-ci.com/codeenigma/ce-deploy.svg?branch=1.x)](https://api.travis-ci.com/codeenigma/ce-deploy.svg?branch=1.x) - A set of Ansible roles and wrapper scripts to deploy (web) applications. + ## Overview The "stack" from this repo is to be installed on a "deploy" server/runner, to be used in conjonction with a CI/CD tool (Jenkins, Gitlab, Travis, ...). -It allows the deploy steps for a given app to be easily customizable at will, and to be stored alongside the codebase of the project. +It allows the deploy steps for a given app to be easily customisable and to be stored alongside the codebase of the project. When triggered from a deployment tool, the stack will clone the codebase and "play" a given deploy playbook from there. diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md index e93a1c33..c7959bcb 100644 --- a/docs/roles/cache_clear/cache_clear-opcache.md +++ b/docs/roles/cache_clear/cache_clear-opcache.md @@ -14,7 +14,7 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - adapter: "" + # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false diff --git a/docs/roles/cli/cachetool.md b/docs/roles/cli/cachetool.md index 9cba4a32..36d0a014 100644 --- a/docs/roles/cli/cachetool.md +++ b/docs/roles/cli/cachetool.md @@ -5,7 +5,8 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version + version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. + ``` diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md index 30d87b11..377ca80c 100644 --- a/docs/roles/deploy_code.md +++ b/docs/roles/deploy_code.md @@ -37,7 +37,7 @@ With this method the live code directory is also the build directory, therefore ## `squashfs` builds Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command). -However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) +However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `squashfs` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/squashfs). Consequently, at the build stage there are two important extra variables to set: @@ -63,10 +63,10 @@ deploy_code: `service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time. -Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory. +Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is named `PROJECTNAME_BUILDTYPE_deploy.sqsh`. [See the code here.](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L80) The previous codebase is *also* packed and copied to the NAS, named `PROJECTNAME_BUILDTYPE_deploy_previous.sqsh` in the same directory. ### Rolling back -Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event. +Rolling back from a bad `squashfs` build means copying the project/branch specific `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting the `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename your `deploy_previous.sqsh` as `PROJECTNAME_BUILDTYPE_deploy.sqsh`, so it is used on an autoscale event. [You can see the actual code that runs on autoscale events here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/mount_sync/templates/init-squashfs.sh.j2) Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md index e93a1c33..c7959bcb 100644 --- a/roles/cache_clear/cache_clear-opcache/README.md +++ b/roles/cache_clear/cache_clear-opcache/README.md @@ -14,7 +14,7 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - adapter: "" + # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false diff --git a/roles/cli/cachetool/README.md b/roles/cli/cachetool/README.md index 9cba4a32..36d0a014 100644 --- a/roles/cli/cachetool/README.md +++ b/roles/cli/cachetool/README.md @@ -5,7 +5,8 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version + version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. + ``` diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md index 30d87b11..377ca80c 100644 --- a/roles/deploy_code/README.md +++ b/roles/deploy_code/README.md @@ -37,7 +37,7 @@ With this method the live code directory is also the build directory, therefore ## `squashfs` builds Because `tarball` is very slow, we have a second method using [`squashfs`](https://github.com/plougher/squashfs-tools). This filesystem is designed for packing and compressing files into read-only images - initially to deploy to removable media - that can simply be mounted, similar to a macOS Apple Disk Image (DWG) file. It is both faster to pack than a tarball *and* instant to deploy (it's just a `mount` command). -However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `mount_sync` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/mount_sync) +However, the build process is more complex. Because mounted `squashfs` images are read only, we cannot build over them as we do in other types of build. [We alter the build path variables in the `_init` role](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/_init/tasks/main.yml#L25) so the build happens in a separate place and then in the `cleanup.yml` we pack the built code into an image ready to be deployed. Again, because the images are read-only mounts, the live site needs to be *unmounted* with an `umount` command and then remounted with a `mount` command to be completely deployed. This requires the `ce-deploy` user to have extra `sudo` permissions, which is handled by [the `squashfs` role in `ce-provision`](https://github.com/codeenigma/ce-provision/tree/1.x/roles/squashfs). Consequently, at the build stage there are two important extra variables to set: @@ -63,10 +63,10 @@ deploy_code: `service_action` is whether `ce-deploy` should restart the services in the list of stop them, unmount and remount the image and start them again. The latter is the only "safe" way to deploy, but results in a second or two of down time. -Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is always named `deploy.sqsh`. The previous codebase is *also* packed and copied to the NAS, named `deploy_previous.sqsh` in the same directory. +Finally, as with the `tarball` method, the packed image is copied up to the NAS to be available to all future servers and is named `PROJECTNAME_BUILDTYPE_deploy.sqsh`. [See the code here.](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/deploy_code/tasks/cleanup.yml#L80) The previous codebase is *also* packed and copied to the NAS, named `PROJECTNAME_BUILDTYPE_deploy_previous.sqsh` in the same directory. ### Rolling back -Rolling back from a bad `squashfs` build means copying `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename `deploy_previous.sqsh` as `deploy.sqsh`, so it is used on an autoscale event. +Rolling back from a bad `squashfs` build means copying the project/branch specific `deploy_previous.sqsh` down from the NAS to a sensible location in the `ce-deploy` user's home directory, unmounting the current image and mounting the `deploy_previous.sqsh` in its place. Once you've done that, to ensure future autoscaling events do not load the bad code, on the NAS you will need to rename `deploy.sqsh` to something else (or delete it entirely if you're sure you don't want it) and rename your `deploy_previous.sqsh` as `PROJECTNAME_BUILDTYPE_deploy.sqsh`, so it is used on an autoscale event. [You can see the actual code that runs on autoscale events here.](https://github.com/codeenigma/ce-provision/blob/1.x/roles/mount_sync/templates/init-squashfs.sh.j2) Same as with the `tarball` method, as long as the `database_backup` is using the `rolling` method then the "roll back" database will still exist and the credentials will be correct in the `deploy_previous.sqsh` image. Again, if the backup method is `dump` then you will need to inspect [the `mysql_backup.dumps_directory` variable](https://github.com/codeenigma/ce-deploy/blob/1.x/roles/database_backup/database_backup-mysql/defaults/main.yml#L4) to see where the backup was saved in order to restore it. diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml index c8d75bd7..22f9e55e 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml @@ -1,11 +1,14 @@ --- - -- set_fact: +- ansible.builtin.set_fact: _admin_user: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-user chars=ascii_letters') }}" -- set_fact: +- ansible.builtin.set_fact: _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}" - name: Reset admin username. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" - name: Reset admin password. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml index c2adafdf..dad5955a 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml @@ -1,16 +1,16 @@ --- # For some unknown reason, updating the field_user_data table did not work reliably, # nor did drush upwd for the password. -# Loading the user directly is akward, but at least means we don't bypass entity update. +# Loading the user directly is awkward, but at least means we don't bypass entity update. - name: Reset admin username. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.username | length > 1 - name: Reset admin password. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: From 09e86396d984ed6d8fbe79142c9bf373ec39eac2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 3 Feb 2023 12:56:57 +0100 Subject: [PATCH 42/73] Drush refactor pr 1.x (#231) * Fixing sync jobs. * Fixing bad task title. --- .../database_apply-drupal7/tasks/ctools.yml | 8 +++-- .../database_sync-mysql/tasks/main.yml | 2 +- .../database_sync-mysql/tasks/sync.yml | 32 ++++++++++--------- roles/sync/database_sync/tasks/main.yml | 2 +- .../cache_clear-drupal8/tasks/main.yml | 13 ++++++++ .../database_apply-drupal7/tasks/main.yml | 2 +- .../database_apply-drupal8/tasks/main.yml | 13 ++++++++ roles/sync/files_sync/tasks/main.yml | 2 +- 8 files changed, 53 insertions(+), 21 deletions(-) diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index ce4db921..194226fa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -1,8 +1,12 @@ --- - name: Check if Ctools module is enabled. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} &&{{ drush_bin }}pm-info ctools | grep ': enabled' | wc -l" + ansible.builtin.shell: + cmd: "{{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" register: ctools_enabled - name: Revert Drupal configuration from Ctools. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} &&{{ drush_bin }}-y ctools-export-revert --all" + ansible.builtin.command: + cmd: "{{ drush_bin }} -y ctools-export-revert --all" + chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: ctools_enabled.stdout == "1" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/main.yml b/roles/sync/database_sync/database_sync-mysql/tasks/main.yml index 11a47d5a..63edc494 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Sync database. - include_tasks: "sync.yml" + ansible.builtin.include_tasks: "sync.yml" with_items: "{{ mysql_sync.databases }}" loop_control: loop_var: database diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 98eca0f3..29100ab5 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -36,11 +36,11 @@ - database.target.asg | length > 0 - name: Register remote dump name (from database). - set_fact: + ansible.builtin.set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" - name: Get source last known good build number. - command: + ansible.builtin.command: argv: - "/bin/sh" - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh" @@ -51,17 +51,17 @@ when: database.source.type == 'rolling' - name: Register source database name. - set_fact: + ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}_{{ mysql_sync_source_build_number.stdout }}" when: database.source.type == 'rolling' - name: Register source database name. - set_fact: + ansible.builtin.set_fact: mysql_sync_source_database: "{{ database.source.database }}" when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - database.source.fresh_db @@ -88,11 +88,11 @@ when: not database.source.fresh_db - name: Register tmp target dump name. - set_fact: + ansible.builtin.set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" - name: Get target last known good build number. - command: + ansible.builtin.command: argv: - "/bin/sh" - "{{ _ce_deploy_base_dir }}/scripts/track-get.sh" @@ -103,38 +103,40 @@ when: database.target.type == 'rolling' - name: Register target database name. - set_fact: + ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}" when: database.target.type == 'rolling' - name: Register target database name. - set_fact: + ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}" when: not database.target.type == 'rolling' - name: Fetch dump file. - fetch: + ansible.builtin.fetch: src: "{{ mysql_sync_source_dump_path }}" dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" flat: true delegate_to: "{{ database.source.host }}" - name: Copy dump file to destination. - copy: + ansible.builtin.copy: src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" dest: "{{ mysql_sync_target_dump_path }}" - name: Drop target database. - shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" + ansible.builtin.command: + cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" - name: Recreate target database. - shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" + ansible.builtin.command: + cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + ansible.builtin.shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - name: Remove tmp dump file. - file: + ansible.builtin.file: path: "{{ mysql_sync_target_dump_path }}" state: absent diff --git a/roles/sync/database_sync/tasks/main.yml b/roles/sync/database_sync/tasks/main.yml index a3992186..650a9e0a 100644 --- a/roles/sync/database_sync/tasks/main.yml +++ b/roles/sync/database_sync/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Sync databases. - include_role: + ansible.builtin.include_role: name: "sync/database_sync/database_sync-{{ engine }}" with_items: "{{ database_sync.engines }}" loop_control: diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml index 96c721ee..431366cf 100644 --- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,4 +1,13 @@ --- +# Drupal 8 ships drush with the website code so we need the previous build in the path. +- name: Stash the drush_bin variable. + ansible.builtin.set_fact: + _drush_bin_deploy: "{{ drush_bin }}" + +- name: Update location of drush for the sync cache clear command. + ansible.builtin.set_fact: + drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" + - name: Clear Drupal cache. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" @@ -9,3 +18,7 @@ loop_control: loop_var: site run_once: true + +- name: Restore the drush_bin variable. + ansible.builtin.set_fact: + drush_bin: "{{ _drush_bin_deploy }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml index f520514e..ea4c083b 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Apply Drupal database updates. - ansible.builtin.shell: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index e050af3f..96d9b65f 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -1,4 +1,13 @@ --- +# Drupal 8 ships drush with the website code so we need the previous build in the path. +- name: Stash the drush_bin variable. + ansible.builtin.set_fact: + _drush_bin_deploy: "{{ drush_bin }}" + +- name: Update location of drush for the update/config import commands. + ansible.builtin.set_fact: + drush_bin: "{{ live_symlink_dest }}/vendor/bin/drush" + - name: Apply Drupal database updates. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" @@ -26,3 +35,7 @@ - name: Clear the cache. ansible.builtin.include_role: name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" + +- name: Restore the drush_bin variable. + ansible.builtin.set_fact: + drush_bin: "{{ _drush_bin_deploy }}" diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml index 47c3667a..3107eb24 100644 --- a/roles/sync/files_sync/tasks/main.yml +++ b/roles/sync/files_sync/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Sync files. - include_tasks: "sync.yml" + ansible.builtin.include_tasks: "sync.yml" with_items: "{{ files_sync.directories }}" loop_control: loop_var: files From 3ba2fe09b1004798dce2feee62619b1eafda0409 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 17 Feb 2023 12:54:40 +0100 Subject: [PATCH 43/73] Drush refactor pr 1.x (#234) * Fixing sync jobs. * Fixing bad task title. * Trying to fix D7 drush issues with chdir. --- .../maintenance_mode-drupal-core/tasks/offline.yml | 4 ++-- .../maintenance_mode-drupal-core/tasks/online.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index f8ae276c..2d6a9310 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -10,10 +10,10 @@ - previous_build_number > 0 - project_type == 'drupal8' +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Enable maintenance mode D7. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" - args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index dfc3e3b3..6b127f89 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -9,10 +9,10 @@ when: - project_type == 'drupal8' +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Disable maintenance mode D7. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" - args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" From 75ff9eedf93c0b438d4c33b9775275121fcb8045 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 17 Feb 2023 13:20:10 +0100 Subject: [PATCH 44/73] Drush refactor pr 1.x (#236) * Fixing sync jobs. * Fixing bad task title. * Trying to fix D7 drush issues with chdir. * Moving D7 drush commands back to 'shell'. --- .../database_apply-drupal7/tasks/ctools.yml | 3 ++- .../database_apply-drupal7/tasks/features.yml | 5 +++-- .../database_apply-drupal7/tasks/main.yml | 12 ++++++------ .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 5 +++-- .../cache_clear/cache_clear-drupal7/tasks/main.yml | 3 ++- .../database_apply-drupal7/tasks/main.yml | 3 ++- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 194226fa..8102d3d6 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -1,4 +1,5 @@ --- +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Ctools module is enabled. ansible.builtin.shell: cmd: "{{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" @@ -6,7 +7,7 @@ register: ctools_enabled - name: Revert Drupal configuration from Ctools. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -y ctools-export-revert --all" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: ctools_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index edba46bf..f55795dc 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,8 +1,9 @@ --- +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Features module is enabled. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" + ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" register: features_enabled - name: Revert Drupal configuration from Features. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}" + ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}" when: features_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index fe25bcaa..70b21947 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -1,7 +1,7 @@ --- - +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Install Drupal. - shell: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -12,7 +12,7 @@ when: previous_build_number == 0 or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. - shell: + ansible.builtin.shell: cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" loop_control: @@ -30,7 +30,7 @@ when: previous_build_number > 0 - name: Apply Drupal database updates. - shell: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" @@ -38,7 +38,7 @@ loop_var: site - name: Revert Drupal configuration with Features. - include_tasks: features.yml + ansible.builtin.include_tasks: features.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site @@ -47,7 +47,7 @@ - site.revert_features_command - name: Revert Drupal configuration with Ctools. - include_tasks: ctools.yml + ansible.builtin.include_tasks: ctools.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml index 22f9e55e..add864cf 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml @@ -4,11 +4,12 @@ - ansible.builtin.set_fact: _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}" +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Reset admin username. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" - name: Reset admin password. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml index ab9342ba..fd22fbff 100644 --- a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -1,6 +1,7 @@ --- +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Clear Drupal 7 cache. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml index ea4c083b..eeaa2d73 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml @@ -1,6 +1,7 @@ --- +# For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Apply Drupal database updates. - ansible.builtin.command: + ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" From 8d0226057a6c979856dfd1a2d44110b1cb1f3a91 Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Mon, 6 Mar 2023 19:27:43 +0200 Subject: [PATCH 45/73] Change php detection for fastcgi and cachetool pr 1.x (#238) * change_php_detection_for_fastcgi_and_cachetool * fix_typo * change_php_detection_for_fastcgi_and_cachetool_no_sudo --- roles/cache_clear/cache_clear-opcache/tasks/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index 57feac51..d7c9f6e2 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Get latest php installed ansible.builtin.shell: - cmd: "ls /etc/php/ | tail -1" + cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' register: _php_version - name: Set cachetool adapter. diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index f0f05b28..2e2a49d1 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -17,7 +17,7 @@ block: - name: Get latest php installed ansible.builtin.shell: - cmd: "ls /etc/php/ | tail -1 | sed -e 's/\\.//g'" + cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"' register: _php_version_str - name: Download latest cachetool installer if PHP is 8.1 or newer. From 7971ce73325610df48987a1adc4ac196a2b05a30 Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Tue, 14 Mar 2023 13:45:43 +0200 Subject: [PATCH 46/73] Cachetool bin per site pr 1.x (#243) * cachetool_bin_per_project * cachetool_bin_per_site_fix * cachetool_bin_per_site_fix2 --- roles/_init/tasks/main.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index effa2790..e939def3 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -107,7 +107,15 @@ - name: Define opcache cachetool path. ansible.builtin.set_fact: - cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}" + cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}" + when: deploy_code.mount_type != "squashfs" + +- name: Define opcache cachetool path if SquashFS deploy. + ansible.builtin.set_fact: + cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Ensure we have a cachetool binary. ansible.builtin.import_role: From 31c3197a50a0cba48e5634499b914915bffb7640 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 14 Mar 2023 18:26:40 +0100 Subject: [PATCH 47/73] apcu no longer has an 'all' option. (#245) --- roles/cache_clear/cache_clear-opcache/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index d7c9f6e2..428e9053 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -15,7 +15,7 @@ - name: Clear apcu. command: - cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear all" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear" when: cache_clear_opcache.clear_apcu - name: Clear stats. From b5fed246e214332b105cd2759cea91be2743b7e8 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Mar 2023 10:21:40 +0100 Subject: [PATCH 48/73] Refactoring cachetool path setting to match other squashfs vars. (#247) --- roles/_init/tasks/main.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index e939def3..7f28f3ff 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -21,6 +21,9 @@ - name: Define live_symlink dest. ansible.builtin.set_fact: live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" +- name: Define opcache cachetool path. + ansible.builtin.set_fact: + cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}" # Manipulate variables for SquashFS builds. - name: Define image builds base path. @@ -47,6 +50,12 @@ when: - deploy_code.mount_type is defined - deploy_code.mount_type == "squashfs" +- name: Overwrite cachetool path if SquashFS deploy. + ansible.builtin.set_fact: + cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Gather last known good build directly from symlink. # This can happen: @@ -105,18 +114,6 @@ when: - _project_type_task_result.stat.exists -- name: Define opcache cachetool path. - ansible.builtin.set_fact: - cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}" - when: deploy_code.mount_type != "squashfs" - -- name: Define opcache cachetool path if SquashFS deploy. - ansible.builtin.set_fact: - cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}" - when: - - deploy_code.mount_type is defined - - deploy_code.mount_type == "squashfs" - - name: Ensure we have a cachetool binary. ansible.builtin.import_role: name: cli/cachetool From 459689f9462f3e6087f14abadfb54d575a129ff6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Mar 2023 12:58:44 +0100 Subject: [PATCH 49/73] Cachetool pr 1.x (#249) * Refactoring cachetool path setting to match other squashfs vars. * Refactoring cachetool_bin handling. --- docs/roles/cache_clear/cache_clear-opcache.md | 1 + roles/_init/tasks/main.yml | 10 +++++++++- roles/cache_clear/cache_clear-opcache/README.md | 1 + .../cache_clear/cache_clear-opcache/defaults/main.yml | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/roles/cache_clear/cache_clear-opcache.md b/docs/roles/cache_clear/cache_clear-opcache.md index c7959bcb..465307e2 100644 --- a/docs/roles/cache_clear/cache_clear-opcache.md +++ b/docs/roles/cache_clear/cache_clear-opcache.md @@ -19,6 +19,7 @@ cache_clear_opcache: clear_opcache: true clear_apcu: false clear_stat: false + # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined ``` diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 7f28f3ff..f68be544 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -23,7 +23,14 @@ live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" - name: Define opcache cachetool path. ansible.builtin.set_fact: - cachetool_bin: "{{ cachetool_bin | default('{{ deploy_base_path }}/cachetool.phar') }}" + cachetool_bin: "{{ deploy_base_path }}/cachetool.phar" + when: cache_clear_opcache.cachetool_bin is not defined +- name: Set opcache cachetool path from variable. + ansible.builtin.set_fact: + cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}" + when: + - cache_clear_opcache.cachetool_bin is defined + - cache_clear_opcache.cachetool_bin | length > 0 # Manipulate variables for SquashFS builds. - name: Define image builds base path. @@ -56,6 +63,7 @@ when: - deploy_code.mount_type is defined - deploy_code.mount_type == "squashfs" + - cache_clear_opcache.cachetool_bin is not defined # Gather last known good build directly from symlink. # This can happen: diff --git a/roles/cache_clear/cache_clear-opcache/README.md b/roles/cache_clear/cache_clear-opcache/README.md index c7959bcb..465307e2 100644 --- a/roles/cache_clear/cache_clear-opcache/README.md +++ b/roles/cache_clear/cache_clear-opcache/README.md @@ -19,6 +19,7 @@ cache_clear_opcache: clear_opcache: true clear_apcu: false clear_stat: false + # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined ``` diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 97623e62..222325ce 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -9,3 +9,4 @@ cache_clear_opcache: clear_opcache: true clear_apcu: false clear_stat: false + # cachetool_bin: "/path/to/cachetool.phar" # see _init for paths if undefined From 0b39b64f6e6fe3c430943a54f04cd072a6689a56 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Mar 2023 13:00:25 +0100 Subject: [PATCH 50/73] Ensuring we can prevent features and ctools deployments in Drupal 7. (#240) --- roles/_init/defaults/main.yml | 3 ++- roles/database_apply/database_apply-drupal7/tasks/main.yml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index b38c1e59..00d03c05 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -14,6 +14,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" + revert_ctools: true # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com @@ -25,4 +26,4 @@ drupal: mautic: image_path: "media/images" force_install: false -bin_directory: "/home/{{ deploy_user }}/.bin" \ No newline at end of file +bin_directory: "/home/{{ deploy_user }}/.bin" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 70b21947..6b745c3b 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -44,10 +44,12 @@ loop_var: site when: - previous_build_number > 0 - - site.revert_features_command + - site.revert_features_command | length > 0 - name: Revert Drupal configuration with Ctools. ansible.builtin.include_tasks: ctools.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site + when: + - site.revert_ctools From 3d6172aca2d37d6d3109192030f1f77d45b62e3d Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 15 Mar 2023 13:21:48 +0100 Subject: [PATCH 51/73] Cachetool pr 1.x (#251) * Refactoring cachetool path setting to match other squashfs vars. * Refactoring cachetool_bin handling. * Forgot to remove the default() filter from squashfs var setting. * Refactoring clauses slightly to simplify. --- roles/_init/tasks/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index f68be544..a0d90571 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -24,8 +24,7 @@ - name: Define opcache cachetool path. ansible.builtin.set_fact: cachetool_bin: "{{ deploy_base_path }}/cachetool.phar" - when: cache_clear_opcache.cachetool_bin is not defined -- name: Set opcache cachetool path from variable. +- name: Set opcache cachetool path from variable if provided. ansible.builtin.set_fact: cachetool_bin: "{{ cache_clear_opcache.cachetool_bin }}" when: @@ -57,9 +56,9 @@ when: - deploy_code.mount_type is defined - deploy_code.mount_type == "squashfs" -- name: Overwrite cachetool path if SquashFS deploy. +- name: Overwrite cachetool path if SquashFS deploy and path not provided. ansible.builtin.set_fact: - cachetool_bin: "{{ cachetool_bin | default('{{ build_base_path }}/cachetool.phar') }}" + cachetool_bin: "{{ build_base_path }}/cachetool.phar" when: - deploy_code.mount_type is defined - deploy_code.mount_type == "squashfs" From 1f2e3b66bb060ec7123ccc253eab2fb487b3e588 Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Mon, 20 Mar 2023 18:13:36 +0200 Subject: [PATCH 52/73] improving_old_builds_cleanup (#254) * improving_old_builds_cleanup * improving_old_builds_cleanup_fix --- roles/_init/defaults/main.yml | 2 ++ .../database_backup-mysql/tasks/cleanup-dump.yml | 2 +- .../database_backup-mysql/tasks/cleanup-rolling.yml | 3 +-- .../database_backup-mysql/tasks/cleanup.yml | 3 +-- roles/deploy_code/tasks/cleanup.yml | 10 +++++----- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 00d03c05..971f1b9b 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -27,3 +27,5 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index a9ea9ae2..2609cabe 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -4,5 +4,5 @@ ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index 2bdf29ee..374e65ca 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -1,5 +1,4 @@ --- -# We assume it's safe to only go back 50 build back. - name: Delete mysql databases. community.mysql.mysql_db: name: "{{ database.database }}_{{ item }}" @@ -7,5 +6,5 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index ac71fcbb..89cbf171 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -15,7 +15,6 @@ - ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" -# We assume it's safe to only go back 50 build back. - name: Delete mysql users. community.mysql.mysql_user: name: "{{ database.user }}_{{ item }}" @@ -24,6 +23,6 @@ login_host: "{{ _mysql_host }}" login_user: "{{ _mysql_user }}" login_password: "{{ _mysql_password }}" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} when: mysql_backup.credentials_handling == 'rotate' run_once: true diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index a5da0d87..6f8dc1f6 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -2,14 +2,14 @@ - name: Ensure codebase is writable. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -18,7 +18,7 @@ - name: Ensure permissions are set on builds directory. ansible.builtin.shell: cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 @@ -28,7 +28,7 @@ ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type != "squashfs" @@ -36,7 +36,7 @@ ansible.builtin.file: name: "{{ build_path_prefix }}{{ item }}" state: absent - with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + with_sequence: start={{ [previous_build_number | int - cleanup_history_depth, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.mount_type == "squashfs" From dd1f5cea2f1dd646999e17f607a863d3ee133547 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 12 Apr 2023 10:26:47 +0200 Subject: [PATCH 53/73] Adding escaped backticks to db names to be safe. (#256) --- .../database_backup-mysql/tasks/deploy-rolling.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 71fa7580..88b51523 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -7,10 +7,20 @@ # Note: we don't use the mysql_db Ansible module on purpose. # If database already exists, we want to fail and not override it # with previous build. + # @TODO fix this so we check if the database exists and exit with + # the proper plugin instead of using command. - name: Create new database. - ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE \`{{ _mysql_build_database_name }}\`;" run_once: true +#- name: Create a new database. +# community.mysql.mysql_db: +# name: "{{ _mysql_build_database_name }}" +# state: present +# config_file: "{{ database.credentials_file }}" +# config_overrides_defaults: true +# run_once: true + - name: Populate new database. ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 From df74ac6032a9327da6e0d6c158adf7dd64e2e3cc Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 12 Apr 2023 10:37:47 +0200 Subject: [PATCH 54/73] Mysql db name pr 1.x (#258) * Adding escaped backticks to db names to be safe. * Looks like Ansible auto-escapes backticks. --- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 88b51523..f7f109bc 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -10,7 +10,7 @@ # @TODO fix this so we check if the database exists and exit with # the proper plugin instead of using command. - name: Create new database. - ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE \`{{ _mysql_build_database_name }}\`;" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE `{{ _mysql_build_database_name }}`;" run_once: true #- name: Create a new database. From 3b49dc9775c41abf00ddf6013894a9e27cd46b51 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 12 Apr 2023 10:44:22 +0200 Subject: [PATCH 55/73] Mysql db name pr 1.x (#260) * Adding escaped backticks to db names to be safe. * Looks like Ansible auto-escapes backticks. * Adding backticks to database names for creating MySQL users too. --- roles/database_backup/database_backup-mysql/tasks/deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index 5442230d..4160806c 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -90,12 +90,12 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user for TCP connections. - ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true - name: Create/update mysql user for unix socket connections. - ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `{{ _mysql_build_database_name }}`.* TO '{{ _mysql_build_user_name }}'@'localhost';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 8c30320c1871fd031c4e98555771c6f7ae143a12 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 13 Apr 2023 17:22:37 +0200 Subject: [PATCH 56/73] Mysql db name pr 1.x (#262) * Adding escaped backticks to db names to be safe. * Looks like Ansible auto-escapes backticks. * Adding backticks to database names for creating MySQL users too. * Adding extra mysqldump flags to stop restores failing on RDS. --- roles/_init/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 971f1b9b..5eed81f2 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -3,7 +3,7 @@ # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" -_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" +_mysqldump_params: "--set-gtid-purged=OFF --skip-definer --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" From 15ae9015a5712b1cf9f0512672afcc28430304a2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Apr 2023 12:49:16 +0200 Subject: [PATCH 57/73] Mysql db name pr 1.x (#264) * Adding escaped backticks to db names to be safe. * Looks like Ansible auto-escapes backticks. * Adding backticks to database names for creating MySQL users too. * Adding extra mysqldump flags to stop restores failing on RDS. * Removing ignore_errors. * Updating mysqldump query. --- roles/_init/defaults/main.yml | 3 ++- roles/deploy_code/tasks/cleanup.yml | 2 +- roles/lhci_run/tasks/main.yml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 5eed81f2..2602afd2 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -3,7 +3,8 @@ # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" -_mysqldump_params: "--set-gtid-purged=OFF --skip-definer --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" +# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 6f8dc1f6..27878342 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -128,7 +128,7 @@ - name: Check if we have a mount already. ansible.builtin.shell: cmd: "mount | grep {{ deploy_base_path }}" - ignore_errors: true + failed_when: false register: _deploy_code_mount_check when: - deploy_code.mount_sync is defined diff --git a/roles/lhci_run/tasks/main.yml b/roles/lhci_run/tasks/main.yml index 6e9648ce..e5a596fe 100644 --- a/roles/lhci_run/tasks/main.yml +++ b/roles/lhci_run/tasks/main.yml @@ -2,12 +2,12 @@ - name: Check if 'lhci' is available. ansible.builtin.command: "which lhci" register: _lhci_run_check_result - ignore_errors: true + failed_when: false - name: Check if 'Xvfb' is running. ansible.builtin.shell: "pgrep Xvfb" register: _lhci_run_check_xvfb_result - ignore_errors: true + failed_when: false when: - _lhci_run_check_result.rc == 0 From c7e4a472e96706c39d38d62b24d1ab5a5d895df6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Apr 2023 20:35:03 +0200 Subject: [PATCH 58/73] Bug fixes pr 1.x (#266) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. --- .ansible-lint | 11 ++++++++--- .github/workflows/ce-deploy-test.yml | 2 +- .../cache_clear/cache_clear-opcache/tasks/main.yml | 10 ++++++---- roles/cli/cachetool/tasks/main.yml | 11 ++++++----- roles/cli/drush/tasks/main.yml | 4 +++- .../database_apply-drupal7/tasks/ctools.yml | 4 +++- .../database_apply-drupal7/tasks/features.yml | 4 +++- .../database_apply-drupal7/tasks/main.yml | 6 ++++-- .../database_apply-drupal8/tasks/main.yml | 14 ++++++++------ .../database_backup-mysql/tasks/deploy-dump.yml | 4 +++- .../database_backup-mysql/tasks/deploy-rolling.yml | 4 +++- .../database_backup-mysql/tasks/revert-dump.yml | 4 +++- roles/deploy_code/tasks/cleanup.yml | 4 +++- .../database_sync-mysql/tasks/sync.yml | 8 ++++++-- 14 files changed, 60 insertions(+), 30 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 7a9eb05b..b6256e0b 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,4 +1,9 @@ skip_list: - - '204' - - '301' - - '701' + - '204' # Lines should be no longer than 160 chars + - '301' # Commands should not change things if nothing needs doing + - '701' # 701 No 'galaxy_info' found + - unnamed-task + - risky-file-permissions + - no-jinja-nesting + - command-instead-of-shell # some Drupal 7 drush commands require shell + - command-instead-of-module diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml index 141fc8e7..c6429842 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-test.yml @@ -10,7 +10,7 @@ jobs: # Name the Job name: Run tests against Ansible code base # Set the type of machine to run on - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: # Checks out a copy of your repository on the ubuntu-latest machine diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index 428e9053..97f91ed4 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -1,7 +1,9 @@ --- - name: Get latest php installed ansible.builtin.shell: - cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' + cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1' + args: + executable: /bin/bash register: _php_version - name: Set cachetool adapter. @@ -9,16 +11,16 @@ _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('--fcgi=127.0.0.1:90' + _php_version.stdout | replace('.','')) }}" - name: Clear opcache. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset" when: cache_clear_opcache.clear_opcache - name: Clear apcu. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear" when: cache_clear_opcache.clear_apcu - name: Clear stats. - command: + ansible.builtin.command: cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear" when: cache_clear_opcache.clear_stat diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 2e2a49d1..c0cbce61 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -14,10 +14,15 @@ - deploy_operation == 'deploy' - name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility + when: + - deploy_operation == 'deploy' + - cachetool.version | length == 0 block: - name: Get latest php installed ansible.builtin.shell: - cmd: 'ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"' + cmd: 'set -o pipefail && ls -1 /etc/php/ | while read ver; do if [ -d "/etc/php/$ver/fpm" ]; then echo "$ver"; fi; done | tail -1 | sed -e "s/\\.//g"' + args: + executable: /bin/bash register: _php_version_str - name: Download latest cachetool installer if PHP is 8.1 or newer. @@ -69,10 +74,6 @@ when: - _php_version_str.stdout | int < 71 - when: - - deploy_operation == 'deploy' - - cachetool.version | length == 0 - - name: "Download the specified {{ cachetool.version }} cachetool version installer." ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml index 5c73be51..0836c91f 100644 --- a/roles/cli/drush/tasks/main.yml +++ b/roles/cli/drush/tasks/main.yml @@ -16,7 +16,9 @@ register: drush_global_directory - name: Check if installed Drush version match. - ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + ansible.builtin.shell: "set -o pipefail && {{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + args: + executable: /bin/bash register: drush_global_version when: - deploy_operation == 'deploy' diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 8102d3d6..518996ab 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -2,8 +2,10 @@ # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Ctools module is enabled. ansible.builtin.shell: - cmd: "{{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" + cmd: "set -o pipefail && {{ drush_bin }} pm-info ctools | grep ': enabled' | wc -l" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: + executable: /bin/bash register: ctools_enabled - name: Revert Drupal configuration from Ctools. diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index f55795dc..2b4357eb 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,7 +1,9 @@ --- # For some reason D7 drush doesn't respect 'chdir' with command, using shell instead. - name: Check if Features module is enabled. - ansible.builtin.shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" + ansible.builtin.shell: "set -o pipefail && cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" + args: + executable: /bin/bash register: features_enabled - name: Revert Drupal configuration from Features. diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 6b745c3b..d64364c4 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -12,8 +12,10 @@ when: previous_build_number == 0 or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. - ansible.builtin.shell: - cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + ansible.builtin.file: + path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + state: directory + mode: '0755' with_items: "{{ drupal.sites }}" loop_control: loop_var: site diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 16f0417b..4dc66a3a 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Fix file permissions for settings.php. - file: + ansible.builtin.file: state: file path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" owner: "{{ www_user }}" @@ -14,7 +14,7 @@ - previous_build_number == 0 - name: Install Drupal. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -25,8 +25,10 @@ when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. - shell: - cmd: "chmod 755 {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + ansible.builtin.file: + path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + state: directory + mode: '0755' with_items: "{{ drupal.sites }}" loop_control: loop_var: site @@ -48,7 +50,7 @@ when: previous_build_number > 0 - name: Apply Drupal database updates. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" @@ -59,7 +61,7 @@ when: site.config_import_command != 'deploy' - name: Import configuration. - command: + ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 80a6e5ef..9f6f6959 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -23,6 +23,8 @@ run_once: true - name: Take a database dump. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index f7f109bc..aec45596 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -22,6 +22,8 @@ # run_once: true - name: Populate new database. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index f914ee78..80898e7d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,7 @@ --- - name: Revert database from dump. - ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + args: + executable: /bin/bash when: previous_build_number > 0 run_once: true diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 27878342..db2bb9bb 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -127,7 +127,9 @@ - name: Check if we have a mount already. ansible.builtin.shell: - cmd: "mount | grep {{ deploy_base_path }}" + cmd: "set -o pipefail && mount | grep {{ deploy_base_path }}" + args: + executable: /bin/bash failed_when: false register: _deploy_code_mount_check when: diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 29100ab5..c695982c 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -61,7 +61,9 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + args: + executable: /bin/bash delegate_to: "{{ database.source.host }}" when: - database.source.fresh_db @@ -133,7 +135,9 @@ cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - ansible.builtin.shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" + args: + executable: /bin/bash - name: Remove tmp dump file. ansible.builtin.file: From 5b28bdd1ea6634d1b3336c9e7135a80ef3d23689 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 19 Apr 2023 19:30:27 +0200 Subject: [PATCH 59/73] Bug fixes pr 1.x (#268) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. * Fixing D7 ctools behaviour to match features handling. --- docs/roles/_init.md | 5 +++++ roles/_init/README.md | 5 +++++ roles/_init/defaults/main.yml | 2 +- roles/database_apply/database_apply-drupal7/tasks/ctools.yml | 2 +- roles/database_apply/database_apply-drupal7/tasks/main.yml | 3 ++- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 1c7e835e..9cb9e024 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least. # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: @@ -23,6 +24,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com @@ -35,6 +37,9 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 + ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index 1c7e835e..9cb9e024 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -12,6 +12,7 @@ These variables **must** be set in the `deploy/common.yml` file, at least. # this will ensure defaults to other roles too. # If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +# for MySQL CE you might want to add '--set-gtid-purged=OFF --skip-definer' here _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: @@ -23,6 +24,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com @@ -35,6 +37,9 @@ mautic: image_path: "media/images" force_install: false bin_directory: "/home/{{ deploy_user }}/.bin" +# Number of dumps/db to look up for cleanup. +cleanup_history_depth: 50 + ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 2602afd2..9a955f3e 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -15,7 +15,7 @@ drupal: # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" - revert_ctools: true + revert_ctools_command: "ctools-export-revert --all" # End Drupal 7 variables sanitize_command: "sql-sanitize" base_url: https://www.example.com diff --git a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml index 518996ab..608e1a2f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/ctools.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/ctools.yml @@ -10,6 +10,6 @@ - name: Revert Drupal configuration from Ctools. ansible.builtin.shell: - cmd: "{{ drush_bin }} -y ctools-export-revert --all" + cmd: "{{ drush_bin }} -y {{ site.revert_ctools_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: ctools_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index d64364c4..4f476253 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -54,4 +54,5 @@ loop_control: loop_var: site when: - - site.revert_ctools + - previous_build_number > 0 + - site.revert_ctools_command | length > 0 From ce46e32fe437daf6680beee5255d58df148e09ed Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 20 Apr 2023 08:25:20 +0200 Subject: [PATCH 60/73] Making sync paths build unique. --- .../database_sync/database_sync-mysql/tasks/sync.yml | 4 ++-- roles/sync/files_sync/tasks/sync.yml | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c695982c..9b949f27 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -37,7 +37,7 @@ - name: Register remote dump name (from database). ansible.builtin.set_fact: - mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" + mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.bz2" - name: Get source last known good build number. ansible.builtin.command: @@ -91,7 +91,7 @@ - name: Register tmp target dump name. ansible.builtin.set_fact: - mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" + mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2" - name: Get target last known good build number. ansible.builtin.command: diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index e2160008..58ca4cd0 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -1,7 +1,11 @@ --- +- name: Register file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}" + - name: Create a temporary directory for source files on localhost. ansible.builtin.file: - path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + path: "{{ file_sync_path }}" state: directory owner: "{{ deploy_user }}" group: "{{ deploy_user }}" @@ -10,12 +14,12 @@ - name: Copy the source files onto the deploy server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/" delegate_to: "localhost" run_once: true - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.command: - cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" delegate_to: "localhost" run_once: true From aa9d3b0329852029f4a1ecbbfdac34fc40dbb6a3 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 21 Apr 2023 17:47:19 +0200 Subject: [PATCH 61/73] Adding --host parameter to build.sh for pre-deploy host checking with Ansible. (#271) --- scripts/_common.sh | 24 ++++++++++++++++++++++++ scripts/build.sh | 4 ++++ scripts/host-check.yml | 15 +++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 scripts/host-check.yml diff --git a/scripts/_common.sh b/scripts/_common.sh index 3715c97a..63e89bb9 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -9,6 +9,7 @@ export ANSIBLE_CONFIG="$OWN_DIR/ansible.cfg" TARGET_DEPLOY_REPO="" TARGET_DEPLOY_PLAYBOOK="" TARGET_DEPLOY_BRANCH="" +TARGET_DEPLOY_HOST="" PREVIOUS_BUILD_NUMBER="" CURRENT_BUILD_NUMBER="" ANSIBLE_EXTRA_VARS="" @@ -49,6 +50,10 @@ parse_options(){ shift TARGET_DEPLOY_PLAYBOOK="$1" ;; + "--host") + shift + TARGET_DEPLOY_HOST="$1" + ;; "--build-number") shift CURRENT_BUILD_NUMBER="$1" @@ -146,6 +151,25 @@ cleanup_build_tmp_dir(){ fi } +# Call Ansible playbook to ensure host exists. +ansible_host_check(){ + if [ -n "$TARGET_DEPLOY_HOST" ]; then + ANSIBLE_BIN=$(command -v ansible-playbook) + ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" + if [ "$DRY_RUN" = "yes" ]; then + ANSIBLE_CMD="$ANSIBLE_CMD --check" + fi + if [ "$VERBOSE" = "yes" ]; then + ANSIBLE_CMD="$ANSIBLE_CMD -vvvv" + fi + if [ -n "$BOTO_PROFILE" ]; then + export AWS_PROFILE="$BOTO_PROFILE" + fi + $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS" + return $? + fi +} + # Trigger actual Ansible job. # $1 (string) # Operation to perform. diff --git a/scripts/build.sh b/scripts/build.sh index 5f7fa21b..233ca764 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -19,6 +19,7 @@ usage(){ echo '--branch: The branch to deploy.' echo '' echo 'Available options:' + echo '--host: Valid Ansible hostname, if you want to run a host check. Can also be a group name.' echo '--ansible-extra-vars: Variable to pass as --extra-vars arguments to ansible-playbook. Make sure to escape them properly.' echo '--previous-stable-build-number: an incremental build number that ' echo '--dry-run: Do not perform any action but run the playbooks in --check mode.' @@ -94,6 +95,9 @@ fi # Get Ansible defaults. get_ansible_defaults_vars +# Optionally carry out a host check. +ansible_host_check + # From this point on, we want to trigger the "revert" if anything fails. ANSIBLE_BUILD_RESULT=1 # Trigger deploy. diff --git a/scripts/host-check.yml b/scripts/host-check.yml new file mode 100644 index 00000000..980c63cb --- /dev/null +++ b/scripts/host-check.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + become: false + tasks: + - name: Ensure the hostname check variable is empty. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "" + - name: Check to see if the Ansible host or hostgroup exists. + ansible.builtin.set_fact: + _ce_deploy_ansible_host_check: "{{ item }}" + with_inventory_hostnames: + - "{{ _deploy_host }}" + - ansible.builtin.fail: + msg: "Host does not exist!" + when: _ce_deploy_ansible_host_check | length == 0 From fd7c17d4a002f3e9cec161bea9c0e6d992f55a9f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 25 Apr 2023 10:04:31 +0200 Subject: [PATCH 62/73] Making syncs safer and more efficient. (#273) --- .../database_sync-mysql/defaults/main.yml | 1 + .../database_sync-mysql/tasks/sync.yml | 20 ++++++++++++++++--- roles/sync/files_sync/defaults/main.yml | 3 +++ roles/sync/files_sync/tasks/sync.yml | 17 ++++++++++++++-- 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 0ab99e98..48c02b8b 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,7 @@ --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 9b949f27..7815868f 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -104,12 +104,12 @@ delegate_to: localhost when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target rolling database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}_{{ mysql_sync_target_build_number.stdout }}" when: database.target.type == 'rolling' -- name: Register target database name. +- name: Register target static database name. ansible.builtin.set_fact: mysql_sync_target_database: "{{ database.target.database }}" when: not database.target.type == 'rolling' @@ -139,11 +139,25 @@ args: executable: /bin/bash -- name: Remove tmp dump file. +- name: Delete temporary dump file on target. ansible.builtin.file: path: "{{ mysql_sync_target_dump_path }}" state: absent +- name: Delete temporary dump file on source. + ansible.builtin.file: + path: "{{ mysql_sync_source_dump_path }}" + state: absent + delegate_to: "{{ database.source.host }}" + +- name: Delete temporary dump file on deploy server. + ansible.builtin.file: + path: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" + state: absent + delegate_to: localhost + when: + - mysql_sync.cleanup + - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml index 5b65b1fb..4e222b40 100644 --- a/roles/sync/files_sync/defaults/main.yml +++ b/roles/sync/files_sync/defaults/main.yml @@ -1,5 +1,8 @@ --- files_sync: + unique_workspace: false # set to true to grab a complete full set of files every sync + # Generally speaking you will *not* want to clean up after file syncs, as leaving the files there makes the next rsync far quicker. + cleanup: false # set to true to delete the synced files after a sync directories: - source: # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml index 58ca4cd0..516059c2 100644 --- a/roles/sync/files_sync/tasks/sync.yml +++ b/roles/sync/files_sync/tasks/sync.yml @@ -1,7 +1,12 @@ --- - name: Register file sync location. + ansible.builtin.set_fact: + file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + +- name: Register unique file sync location. ansible.builtin.set_fact: file_sync_path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}_{{ build_number }}" + when: files_sync.unique_workspace - name: Create a temporary directory for source files on localhost. ansible.builtin.file: @@ -15,11 +20,19 @@ - name: Copy the source files onto the deploy server. ansible.builtin.command: cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ file_sync_path }}/" - delegate_to: "localhost" + delegate_to: localhost run_once: true - name: Copy the source files from the deploy server onto the destination server. ansible.builtin.command: cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ file_sync_path }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" - delegate_to: "localhost" + delegate_to: localhost run_once: true + +- name: Delete synced files on deploy server. + ansible.builtin.file: + path: "{{ file_sync_path }}" + state: absent + delegate_to: localhost + when: + - files_sync.cleanup From ac8b1144dee9189b6830e4f901542f54d3ca02a3 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 11 May 2023 14:09:25 +0200 Subject: [PATCH 63/73] Bug fixes pr 1.x (#278) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. * Fixing D7 ctools behaviour to match features handling. * Jinja2 template looking for settings.php using wrong path. * Fixing config_generate for Drupal. * Making host checking more robust. * Removing variable declaration that will break host check. * Allow us to use deploy_code with a completely custom build and do nothing. --- docs/roles/sync/database_sync/database_sync-mysql.md | 1 + .../config_generate-drupal7/tasks/settings.yml | 2 +- roles/deploy_code/deploy_code-custom/tasks/main.yml | 3 +++ roles/sync/database_sync/database_sync-mysql/README.md | 1 + scripts/_common.sh | 6 +++--- scripts/build.sh | 10 ++++++++-- 6 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 roles/deploy_code/deploy_code-custom/tasks/main.yml diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index cf7e5311..cbe1a353 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f8e3b55e..55d9dee7 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -19,6 +19,6 @@ src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php.j2" + - "{{ playbook_dir }}/{{ site.folder }}/{{ build_type }}.settings.php.j2" - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.settings.php" - "settings.php.j2" diff --git a/roles/deploy_code/deploy_code-custom/tasks/main.yml b/roles/deploy_code/deploy_code-custom/tasks/main.yml new file mode 100644 index 00000000..6f34c92f --- /dev/null +++ b/roles/deploy_code/deploy_code-custom/tasks/main.yml @@ -0,0 +1,3 @@ +--- + +# Nothing to do here. \ No newline at end of file diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index cf7e5311..cbe1a353 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -6,6 +6,7 @@ Sync MySQL databases between environments. --- mysql_sync: mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here + cleanup: true # if false leaves tmp database dump on deploy server for debugging purposes databases: - source: # Name of the database to take a dump from. diff --git a/scripts/_common.sh b/scripts/_common.sh index 63e89bb9..74ba9a60 100755 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -156,9 +156,6 @@ ansible_host_check(){ if [ -n "$TARGET_DEPLOY_HOST" ]; then ANSIBLE_BIN=$(command -v ansible-playbook) ANSIBLE_CMD="$ANSIBLE_BIN $OWN_DIR/scripts/host-check.yml" - if [ "$DRY_RUN" = "yes" ]; then - ANSIBLE_CMD="$ANSIBLE_CMD --check" - fi if [ "$VERBOSE" = "yes" ]; then ANSIBLE_CMD="$ANSIBLE_CMD -vvvv" fi @@ -167,6 +164,9 @@ ansible_host_check(){ fi $ANSIBLE_CMD --extra-vars "{_deploy_host: $TARGET_DEPLOY_HOST}" --extra-vars "$ANSIBLE_DEFAULT_EXTRA_VARS" --extra-vars "$ANSIBLE_EXTRA_VARS" return $? + # No host to check provided, just return a clean exit code. + else + return 0 fi } diff --git a/scripts/build.sh b/scripts/build.sh index 233ca764..93f36887 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -95,8 +95,14 @@ fi # Get Ansible defaults. get_ansible_defaults_vars -# Optionally carry out a host check. +# Optionally carry out a host check if --host is provided. ansible_host_check +ANSIBLE_HOST_CHECK_RESULT=$? +# Exit early if host not found. +if [ -n "$ANSIBLE_HOST_CHECK_RESULT" ] && [ "$ANSIBLE_HOST_CHECK_RESULT" != 0 ]; then + echo "ce-deploy failed to find the host. Aborting." + exit 1 +fi # From this point on, we want to trigger the "revert" if anything fails. ANSIBLE_BUILD_RESULT=1 @@ -113,5 +119,5 @@ if [ -n "$ANSIBLE_BUILD_RESULT" ] && [ "$ANSIBLE_BUILD_RESULT" = 0 ]; then exit 0 fi # Failed somehow. Normally unreachable in strict mode. -echo "Something went wrong. Please fill a bug report against ce-deploy." +echo "Something went unexpectedly wrong with ce-deploy. Please file a bug report - https://github.com/codeenigma/ce-deploy/issues/new" exit 1 \ No newline at end of file From 430b731f422f9c9e486de9939aeeb190533c62be Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 12 May 2023 14:51:19 +0200 Subject: [PATCH 64/73] Bug fixes pr 1.x (#282) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. * Fixing D7 ctools behaviour to match features handling. * Jinja2 template looking for settings.php using wrong path. * Fixing config_generate for Drupal. * Making host checking more robust. * Removing variable declaration that will break host check. * Allow us to use deploy_code with a completely custom build and do nothing. * Make PHP cachetool install optional. --- docs/roles/_init.md | 1 + roles/_init/README.md | 1 + roles/_init/defaults/main.yml | 1 + roles/_init/tasks/main.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index 9cb9e024..08ffe301 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -39,6 +39,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index 9cb9e024..08ffe301 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -39,6 +39,7 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9a955f3e..ac06e2a7 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -30,3 +30,4 @@ mautic: bin_directory: "/home/{{ deploy_user }}/.bin" # Number of dumps/db to look up for cleanup. cleanup_history_depth: 50 +install_php_cachetool: true # set to false if you don't need cachetool, e.g. for a nodejs app diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a0d90571..c054e8fb 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -124,3 +124,4 @@ - name: Ensure we have a cachetool binary. ansible.builtin.import_role: name: cli/cachetool + when: install_php_cachetool From 4d211b5a600505c86f3c803a9bf6baec5493b325 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 17 May 2023 15:47:49 +0200 Subject: [PATCH 65/73] Bug fixes pr 1.x (#284) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. * Fixing D7 ctools behaviour to match features handling. * Jinja2 template looking for settings.php using wrong path. * Fixing config_generate for Drupal. * Making host checking more robust. * Removing variable declaration that will break host check. * Allow us to use deploy_code with a completely custom build and do nothing. * Make PHP cachetool install optional. * Fixing bug introduced by failed_when - need to check for return code on mount points. --- roles/deploy_code/tasks/cleanup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index db2bb9bb..b3a4f132 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -150,7 +150,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Stop any services that might be keeping the loop device busy. @@ -166,7 +166,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - deploy_code.services | length > 0 - name: Unmount existing SquashFS image. @@ -177,7 +177,7 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _deploy_code_mount_check is succeeded + - _deploy_code_mount_check.rc == 0 - name: Mount new SquashFS image. ansible.builtin.command: From 7dbc30caba2ce5bfa6c2719b151d6714137bba46 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 25 May 2023 15:34:43 +0200 Subject: [PATCH 66/73] Bug fixes pr 1.x (#287) * Fixing linting so CI can work. * More linting fixes. * Downgrading Ubuntu due to Docker issues with ce-dev and latest. * Making pipefail shell commands use /bin/bash. * Adding pipefail code to cachetool installer block. * Adding executable for drush install checker. * Fixing D7 ctools behaviour to match features handling. * Jinja2 template looking for settings.php using wrong path. * Fixing config_generate for Drupal. * Making host checking more robust. * Removing variable declaration that will break host check. * Allow us to use deploy_code with a completely custom build and do nothing. * Make PHP cachetool install optional. * Fixing bug introduced by failed_when - need to check for return code on mount points. * Moving where opcache clears get called so they can be excluded if necessary. --- roles/_meta/deploy-drupal8/tasks/main.yml | 4 ++-- roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 ----- roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 ----- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 6ce31d4f..3e66875b 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -20,14 +20,14 @@ name: database_backup - import_role: name: config_generate +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: database_apply - import_role: name: sanitize/admin_creds - import_role: name: live_symlink -- import_role: - name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 4f476253..3012032f 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -26,11 +26,6 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 4dc66a3a..c649826c 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -44,11 +44,6 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' -- name: Clear the opcache. - ansible.builtin.include_role: - name: cache_clear/cache_clear-opcache - when: previous_build_number > 0 - - name: Apply Drupal database updates. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 5fefc81fa1387528e145402752706b41810d4726 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 13:26:09 +0300 Subject: [PATCH 67/73] db_import_speed_up --- .../tasks/revert-dump.yml | 15 +++++++++---- .../database_sync-mysql/tasks/sync.yml | 21 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 80898e7d..0445c44e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,7 +1,14 @@ --- -- name: Revert database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" - args: - executable: /bin/bash +- name: Unpack dump file. + ansible.builtin.unarchive: + src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + run_once: true when: previous_build_number > 0 + +- name: Revert database from dump. + ansible.builtin.command: + cmd: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" + delegate_to: localhost run_once: true + when: previous_build_number > 0 diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 7815868f..d5a7f515 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -93,6 +93,10 @@ ansible.builtin.set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2" +- name: Register tmp target unpacked dump name. + ansible.builtin.set_fact: + mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql" + - name: Get target last known good build number. ansible.builtin.command: argv: @@ -126,6 +130,16 @@ src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" dest: "{{ mysql_sync_target_dump_path }}" +- name: Unpack dump file. + ansible.builtin.unarchive: + src: "{{ mysql_sync_target_dump_path }}" + dest: "{{ mysql_sync_target_dump_unpacked_path }}" + +- name: Delete temporary dump file on target. + ansible.builtin.file: + path: "{{ mysql_sync_target_dump_path }}" + state: absent + - name: Drop target database. ansible.builtin.command: cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" @@ -135,13 +149,12 @@ cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - ansible.builtin.shell: "set -o pipefail && bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - args: - executable: /bin/bash + ansible.builtin.command: + cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}" - name: Delete temporary dump file on target. ansible.builtin.file: - path: "{{ mysql_sync_target_dump_path }}" + path: "{{ mysql_sync_target_dump_unpacked_path }}" state: absent - name: Delete temporary dump file on source. From 69579e45744c95b5643330964febb09d273316bf Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 13:34:56 +0300 Subject: [PATCH 68/73] db_import_speed_up_fixed --- .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 0445c44e..ee4b852e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -9,6 +9,5 @@ - name: Revert database from dump. ansible.builtin.command: cmd: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" - delegate_to: localhost run_once: true when: previous_build_number > 0 From dbcd09e064ff95b965a1f5bba61e5b44f34174bd Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 14:17:47 +0300 Subject: [PATCH 69/73] db_import_speed_up_fix2 --- .../database_backup-mysql/tasks/revert-dump.yml | 10 ++++++++-- .../database_sync/database_sync-mysql/tasks/sync.yml | 9 +++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index ee4b852e..722887a8 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -7,7 +7,13 @@ when: previous_build_number > 0 - name: Revert database from dump. - ansible.builtin.command: - cmd: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }} < /tmp/{{ database.database }}-{{ previous_build_number }}.sql" + args: + executable: /bin/bash run_once: true when: previous_build_number > 0 + +- name: Delete unpacked dump file. + ansible.builtin.file: + path: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + state: absent diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index d5a7f515..7bfaea0b 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -93,7 +93,7 @@ ansible.builtin.set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2" -- name: Register tmp target unpacked dump name. +- name: Register tmp unpacked target dump name. ansible.builtin.set_fact: mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql" @@ -149,10 +149,11 @@ cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - name: Repopulate database from dump. - ansible.builtin.command: - cmd: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}" + ansible.builtin.shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }} < {{ mysql_sync_target_dump_unpacked_path }}" + args: + executable: /bin/bash -- name: Delete temporary dump file on target. +- name: Delete temporary unpacked dump file on target. ansible.builtin.file: path: "{{ mysql_sync_target_dump_unpacked_path }}" state: absent From 187e503e60b88f28b0719d851d5369e4e6804cdf Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 14:19:49 +0300 Subject: [PATCH 70/73] db_import_speed_up_fix3 --- .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 722887a8..07cc1530 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -17,3 +17,5 @@ ansible.builtin.file: path: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" state: absent + run_once: true + when: previous_build_number > 0 From d284adb8be966fdfe6d4c33435623a2aca24f646 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 16:47:16 +0300 Subject: [PATCH 71/73] db_import_speed_up_fix4 --- .../database_backup/database_backup-mysql/tasks/revert-dump.yml | 1 + roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 07cc1530..38522af3 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -3,6 +3,7 @@ ansible.builtin.unarchive: src: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" dest: "/tmp/{{ database.database }}-{{ previous_build_number }}.sql" + remote_src: true run_once: true when: previous_build_number > 0 diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 7bfaea0b..11b50d12 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -134,6 +134,7 @@ ansible.builtin.unarchive: src: "{{ mysql_sync_target_dump_path }}" dest: "{{ mysql_sync_target_dump_unpacked_path }}" + remote_src: true - name: Delete temporary dump file on target. ansible.builtin.file: From 5c845f83f689b8b0a9d67b8d43736692f9c51490 Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 18:46:31 +0300 Subject: [PATCH 72/73] db_import_speed_up_fix5_and_gzip --- .../database_sync-mysql/defaults/main.yml | 2 + .../database_sync-mysql/tasks/sync.yml | 63 ++++++++++++++++--- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 48c02b8b..72add57e 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -24,6 +24,8 @@ mysql_sync: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. asg: "" + # Default is bzip2, set to true if you want to use gzip. + use_gzip: false target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 11b50d12..e4b040a0 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -35,10 +35,16 @@ - database.target.asg is defined - database.target.asg | length > 0 -- name: Register remote dump name (from database). +- name: Register remote dump name (from database) - BZIP2. ansible.builtin.set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.bz2" +- name: Register remote dump name (from database) - GZIP. + ansible.builtin.set_fact: + mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.gz" + when: + - database.source.use_gzip + - name: Get source last known good build number. ansible.builtin.command: argv: @@ -60,7 +66,7 @@ mysql_sync_source_database: "{{ database.source.database }}" when: not database.source.type == 'rolling' -- name: Take a dump from source database. +- name: Take a dump from source database - BZIP2. ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" args: executable: /bin/bash @@ -68,6 +74,15 @@ when: - database.source.fresh_db +- name: Take a dump from source database - GZIP. + ansible.builtin.shell: "set -o pipefail && mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | gzip > {{ mysql_sync_source_dump_path }}" + args: + executable: /bin/bash + delegate_to: "{{ database.source.host }}" + when: + - database.source.fresh_db + - database.source.use_gzip + - name: Find source database host. ansible.builtin.command: cmd: "grep 'host' {{ database.source.credentials_file }}" @@ -89,10 +104,16 @@ delegate_to: "{{ database.source.host }}" when: not database.source.fresh_db -- name: Register tmp target dump name. +- name: Register tmp target dump name - BZIP2. ansible.builtin.set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.bz2" +- name: Register tmp target dump name - GZIP. + ansible.builtin.set_fact: + mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.gz" + when: + - database.source.use_gzip + - name: Register tmp unpacked target dump name. ansible.builtin.set_fact: mysql_sync_target_dump_unpacked_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql" @@ -118,23 +139,45 @@ mysql_sync_target_database: "{{ database.target.database }}" when: not database.target.type == 'rolling' -- name: Fetch dump file. +- name: Fetch dump file - BZIP2. ansible.builtin.fetch: src: "{{ mysql_sync_source_dump_path }}" dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" flat: true delegate_to: "{{ database.source.host }}" -- name: Copy dump file to destination. +- name: Fetch dump file - GZIP. + ansible.builtin.fetch: + src: "{{ mysql_sync_source_dump_path }}" + dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.gz" + flat: true + delegate_to: "{{ database.source.host }}" + when: + - database.source.use_gzip + +- name: Copy dump file to destination - BZIP2. ansible.builtin.copy: src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.bz2" dest: "{{ mysql_sync_target_dump_path }}" -- name: Unpack dump file. - ansible.builtin.unarchive: - src: "{{ mysql_sync_target_dump_path }}" - dest: "{{ mysql_sync_target_dump_unpacked_path }}" - remote_src: true +- name: Copy dump file to destination - GZIP. + ansible.builtin.copy: + src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.gz" + dest: "{{ mysql_sync_target_dump_path }}" + when: + - database.source.use_gzip + +- name: Unpack dump file - BZIP2. + ansible.builtin.shell: "bzip2 -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" + args: + executable: /bin/bash + +- name: Unpack dump file - GZIP. + ansible.builtin.shell: "gzip -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" + args: + executable: /bin/bash + when: + - database.source.use_gzip - name: Delete temporary dump file on target. ansible.builtin.file: From 0976176094df5484165066a77b1b9d4d0dbfdb3d Mon Sep 17 00:00:00 2001 From: tim Date: Wed, 21 Jun 2023 19:27:22 +0300 Subject: [PATCH 73/73] db_import_speed_up_fix6 --- .../database_sync-mysql/tasks/sync.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index e4b040a0..5be6f8a9 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -42,8 +42,7 @@ - name: Register remote dump name (from database) - GZIP. ansible.builtin.set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}_{{ build_number }}_source.sql.gz" - when: - - database.source.use_gzip + when: database.source.use_gzip - name: Get source last known good build number. ansible.builtin.command: @@ -111,8 +110,7 @@ - name: Register tmp target dump name - GZIP. ansible.builtin.set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}_{{ build_number }}_target.sql.gz" - when: - - database.source.use_gzip + when: database.source.use_gzip - name: Register tmp unpacked target dump name. ansible.builtin.set_fact: @@ -152,8 +150,7 @@ dest: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.gz" flat: true delegate_to: "{{ database.source.host }}" - when: - - database.source.use_gzip + when: database.source.use_gzip - name: Copy dump file to destination - BZIP2. ansible.builtin.copy: @@ -164,8 +161,7 @@ ansible.builtin.copy: src: "{{ _ce_deploy_build_tmp_dir }}/{{ database.target.database }}.sql.gz" dest: "{{ mysql_sync_target_dump_path }}" - when: - - database.source.use_gzip + when: database.source.use_gzip - name: Unpack dump file - BZIP2. ansible.builtin.shell: "bzip2 -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" @@ -176,8 +172,7 @@ ansible.builtin.shell: "gzip -d -c {{ mysql_sync_target_dump_path }} > {{ mysql_sync_target_dump_unpacked_path }}" args: executable: /bin/bash - when: - - database.source.use_gzip + when: database.source.use_gzip - name: Delete temporary dump file on target. ansible.builtin.file: