From 3ccfe51727c35f7cef25216f0ae4d0701d9dabb9 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 20 Jan 2022 10:13:40 +0000 Subject: [PATCH 01/29] Defer nightly backups, disable ASG processes during syncs and run syncs with backup (#94) * Change location of nightly backup script and delegate the cron that runs it to the deploy server, if required (for ASGs). * Set up nightly backup crons in separate files in /etc/cron.d * Can't put site cron files in /etc/cron.d because the deploy user doesn't have perms. * Try and add the ability to sync a site using a nightly backup instead of a fresh DB dump. * Use Ansible modules to look up RDS host and to copy the nightly backup into place. * Delegate PATH setup in db backup cron to localhost. * Shell bad. Command good. But makes it convoluted. Oh well. * Used wrong database name in source database copy. * Try and disable the ReplaceUnhealthy auto scale process during syncs. * Fix Drupal cron roles when deferring to deploy server. --- .../cron_database_backup-mysql/tasks/main.yml | 10 +++ .../tasks/setup.yml | 18 ++++- roles/cron/cron_drupal7/tasks/main.yml | 1 + roles/cron/cron_drupal8/tasks/main.yml | 1 + .../database_sync-mysql/defaults/main.yml | 9 +++ .../database_sync-mysql/tasks/sync.yml | 79 ++++++++++++++++++- 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index 04789df8..ffd14157 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -3,6 +3,16 @@ # screwing the daily backup when using rolling db, we could # add a keep mechanism for backup scripts, like for the dumps themselves. # Nice to have more than anything. +- name: Setup PATH in crontab. + cron: + name: PATH + env: true + job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost + when: + - drupal.defer is defined + - drupal.defer + - include_tasks: setup.yml vars: database: database diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index bdc1534c..40b0c47e 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -25,12 +25,26 @@ - name: Create backup script. template: src: "regular-backups.sh.j2" - dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" mode: 0700 +- name: Define backup cron job command. + set_fact: + _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + +- name: Define backup cron job command if deferred (ASG). + set_fact: + _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\"" + when: + - drupal.defer is defined + - drupal.defer + - drupal.defer_target is defined + - drupal.defer_target | length > 0 + - name: Setup regular backup for MySQL. cron: name: "cron_mysql_{{ database.host }}_{{ database.original.database }}" minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" - job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + job: "{{ _backup_cron_job_command }}" + delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index c1d7b259..48e3e4a0 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -15,6 +15,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -25,3 +32,5 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 81e9b2ce..b8182bc3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -1,4 +1,40 @@ --- +- name: Get database source host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_source_database_host_info + delegate_to: "{{ database.source.host }}" + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. +- name: Get database target host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_target_database_host_info + when: + - database.target.asg is defined + - database.target.asg | length > 0 + +- name: Disable ReplaceUnhealthy autoscale process on source ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Disable ReplaceUnhealthy autoscale process on target ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 + - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" @@ -33,7 +69,30 @@ - name: Take a dump from source database. shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" - when: not database.source.type == 'dump' + when: + - not database.source.type == 'dump' + - database.source.fresh_db + +- name: Find source database host. + ansible.builtin.command: + cmd: "grep 'host' {{ database.source.credentials_file }}" + register: mysql_host_info_grep + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Register source database host. + set_fact: + mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Copy a nightly backup for the source database. + ansible.builtin.copy: + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" + dest: "{{ mysql_sync_source_dump_path }}" + remote_src: true + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db - name: Register tmp target dump name. set_fact: @@ -95,3 +154,21 @@ path: "{{ mysql_sync_target_dump_path }}" state: absent when: not database.target.type == 'dump' + +- name: Enable all autoscale processes on source ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Enable all autoscale processes on target ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 From a54c09cb0ce5b6889e672ba8ea741b71a3da034b Mon Sep 17 00:00:00 2001 From: Dionisio Date: Thu, 10 Mar 2022 14:16:13 +0100 Subject: [PATCH 02/29] Added deploy.yml examples for Drupal 9 and Localgov. Updated tests (#97) --- .github/workflows/ce-deploy-test.yml | 22 ++++++++ ce-dev/ansible/examples/drupal9/deploy.yml | 55 +++++++++++++++++++ ce-dev/ansible/examples/localgov/deploy.yml | 59 +++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 ce-dev/ansible/examples/drupal9/deploy.yml create mode 100644 ce-dev/ansible/examples/localgov/deploy.yml diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml index baa13347..141fc8e7 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-test.yml @@ -59,6 +59,28 @@ jobs: curl https://www.test.local shell: bash + - name: Run a test ce-dev deploy with Drupal 9 + run: | + ce-dev create -p testnine -t drupal9 -d ~/testnine + cd ~/testnine + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testnine.local + shell: bash + + - name: Run a test ce-dev deploy with Localgov + run: | + ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov + cd ~/testlocalgov + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testlocalgov.local + shell: bash + # Builds the table of contents for the docs - name: Documentation (build table of contents) if: ${{ github.event.pull_request.base.ref == '1.x' }} diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml new file mode 100644 index 00000000..deeef9ee --- /dev/null +++ b/ce-dev/ansible/examples/drupal9/deploy.yml @@ -0,0 +1,55 @@ +--- +# Template playbook for a local Drupal9 codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: no + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml new file mode 100644 index 00000000..9844206b --- /dev/null +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -0,0 +1,59 @@ +--- +# Template playbook for a local localgov codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si localgov" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: false + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + - name: Install localgov. + command: + cmd: composer require localgovdrupal/localgov + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file From 223a3b5ecec42997d9e52de690cf7b8c5f9f19ab Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 31 Mar 2022 12:43:36 +0200 Subject: [PATCH 03/29] Adding a new SimpleSAMLphp meta role. (#100) --- roles/_meta/deploy-simplesamlphp/README.md | 9 +++++++++ roles/_meta/deploy-simplesamlphp/tasks/main.yml | 17 +++++++++++++++++ .../deploy_code-simplesamlphp/tasks/main.yml | 1 + 3 files changed, 27 insertions(+) create mode 100644 roles/_meta/deploy-simplesamlphp/README.md create mode 100644 roles/_meta/deploy-simplesamlphp/tasks/main.yml create mode 100644 roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml diff --git a/roles/_meta/deploy-simplesamlphp/README.md b/roles/_meta/deploy-simplesamlphp/README.md new file mode 100644 index 00000000..3d82e671 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/README.md @@ -0,0 +1,9 @@ +# SimpleSAMLphp +Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer. + +This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision: + +* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx + + + diff --git a/roles/_meta/deploy-simplesamlphp/tasks/main.yml b/roles/_meta/deploy-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..7af28fa2 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Default SimpleSAMLphp role. This is suitable for a standalone SimpleSAMLphp installation + +- ansible.builtin.import_role: + name: _init +- ansible.builtin.import_role: + name: deploy_code +- ansible.builtin.import_role: + name: composer +- ansible.builtin.import_role: + name: database_backup +- ansible.builtin.import_role: + name: live_symlink +- ansible.builtin.import_role: + name: cache_clear/cache_clear-opcache +- ansible.builtin.import_role: + name: _exit diff --git a/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..03c03856 --- /dev/null +++ b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml @@ -0,0 +1 @@ +# Nothing to do. \ No newline at end of file From 32843d53a625b68bc3a3f7a8cfcced5c90df3f49 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 13 Apr 2022 13:09:01 +0200 Subject: [PATCH 04/29] Allowing users to set cachetool version properly. (#102) --- roles/cli/cachetool/defaults/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index ad2c6711..5ec1b2d7 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: 8.2.2 \ No newline at end of file + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version \ No newline at end of file diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 5a1969c6..aa27df04 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -13,7 +13,7 @@ when: - deploy_operation == 'deploy' -- name: Download cachetool installer. +- name: Download latest cachetool installer. get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" dest: "{{ cachetool_bin }}" @@ -21,3 +21,14 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists + - cachetool.version == 'latest' + +- name: "Download cachetool version {{ cachetool.version }} installer." + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - deploy_operation == 'deploy' + - not cachetool_global.stat.exists + - cachetool.version != 'latest' From 91ea64efc91984ce88138857898b927b5ffd7b25 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 14 Apr 2022 17:13:09 +0200 Subject: [PATCH 05/29] Deploy ami pr 1.x (#106) * Including an ami.yml for packing AMIs on build finale. * New api_call role, focused on GitLab for now. * Optionally trigger an infra build with an API call. --- roles/api_call/README.md | 9 ++++++++ roles/api_call/defaults/main.yml | 13 +++++++++++ roles/api_call/tasks/main.yml | 35 +++++++++++++++++++++++++++++ roles/deploy_code/defaults/main.yml | 21 +++++++++++++++++ roles/deploy_code/tasks/cleanup.yml | 27 +++++++++++++++++----- roles/deploy_code/tasks/deploy.yml | 10 ++++----- roles/deploy_code/tasks/main.yml | 2 +- 7 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 roles/api_call/README.md create mode 100644 roles/api_call/defaults/main.yml create mode 100644 roles/api_call/tasks/main.yml diff --git a/roles/api_call/README.md b/roles/api_call/README.md new file mode 100644 index 00000000..83d1ab28 --- /dev/null +++ b/roles/api_call/README.md @@ -0,0 +1,9 @@ +# API call + +Making RESTful API calls to other platforms. + + + + + + diff --git a/roles/api_call/defaults/main.yml b/roles/api_call/defaults/main.yml new file mode 100644 index 00000000..091342e8 --- /dev/null +++ b/roles/api_call/defaults/main.yml @@ -0,0 +1,13 @@ +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 \ No newline at end of file diff --git a/roles/api_call/tasks/main.yml b/roles/api_call/tasks/main.yml new file mode 100644 index 00000000..c837df2d --- /dev/null +++ b/roles/api_call/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Ensure variables are empty. + ansible.builtin.set_fact: + _api_call_variables: "" + _api_call_url: "" + +- name: Build HTML escaped variable string. + ansible.builtin.set_fact: + _api_call_variables: "{{ _api_call_variables + ('' if ansible_loop.first else '&') + 'variables' + item }}" + with_items: "{{ api_call.variables }}" + loop_control: + extended: true + when: api_call.variables | length > 0 + +- name: Build anonymous API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{{ _api_call_variables }}" + when: api_call.token | length == 0 + +- name: Build token authenticated API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{% if api_call.token_type == 'trigger' %}token={% else %}private_token={% endif %}{{ api_call.token }}&{{ _api_call_variables }}" + when: api_call.token | length > 0 + +- name: Display URL to call. + ansible.builtin.debug: + msg: "{{ _api_call_url }}" + +- name: Make API call. + ansible.builtin.uri: + url: "{{ _api_call_url }}" + method: "{{ api_call.method }}" + return_content: true + status_code: "{{ api_call.status_codes }}" + register: _api_call_return diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index 19e45eee..d616bf08 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -21,3 +21,24 @@ deploy_code: # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index dc72c752..4e648d25 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -1,13 +1,13 @@ --- - name: Ensure codebase is writable. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on directory. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: @@ -15,13 +15,13 @@ - deploy_code.perms_fix_path | length > 1 - name: Delete codebases. - file: + ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} - name: Create a tarball of the deployed codebases. - command: + ansible.builtin.command: cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined @@ -29,7 +29,7 @@ run_once: true - name: Create destination folder. - file: + ansible.builtin.file: path: "{{ deploy_code.mount_sync }}" state: directory mode: "0755" @@ -39,9 +39,24 @@ run_once: true - name: Move to final destination. - command: + ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 run_once: true + +- name: Trigger an infrastructure rebuild. + ansible.builtin.include_role: + name: api_call + vars: + api_call: + type: "{{ deploy_code.api_call.type }}" + base_url: "{{ deploy_code.api_call.base_url }}" + path: "{{ deploy_code.api_call.path }}" + method: "{{ deploy_code.api_call.method }}" + token: "{{ deploy_code.api_call.token }}" + token_type: "{{ deploy_code.api_call.token_type }}" + variables: "{{ deploy_code.api_call.variables }}" + status_codes: "{{ deploy_code.api_call.status_codes }}" + when: deploy_code.rebuild_infra diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml index 3693eb70..0d15425c 100644 --- a/roles/deploy_code/tasks/deploy.yml +++ b/roles/deploy_code/tasks/deploy.yml @@ -1,6 +1,6 @@ --- - name: Copy project repository. - synchronize: + ansible.posix.synchronize: src: "{{ _ce_deploy_build_dir }}/" dest: "{{ deploy_path }}" archive: true @@ -8,17 +8,17 @@ - "--exclude=.git" - name: Ensure project repository is readable. - file: + ansible.builtin.file: path: "{{ deploy_path }}" state: directory mode: 0755 - name: Project specific tasks. - include_role: + ansible.builtin.include_role: name: "deploy_code/deploy_code-{{ project_type }}" - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ deploy_code.templates }}" @@ -29,7 +29,7 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link diff --git a/roles/deploy_code/tasks/main.yml b/roles/deploy_code/tasks/main.yml index a8021dcf..f4bd05af 100644 --- a/roles/deploy_code/tasks/main.yml +++ b/roles/deploy_code/tasks/main.yml @@ -1,2 +1,2 @@ --- -- include_tasks: "{{ deploy_operation }}.yml" +- ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" From 371b6b9d8b28a908facfe2d80779efd76cb64b40 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 16:20:36 +0200 Subject: [PATCH 06/29] Making the MySQL dump command for routine back-ups less aggressive. (#104) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. --- roles/_init/defaults/main.yml | 1 + .../cron_database_backup-mysql/defaults/main.yml | 1 + .../templates/regular-backups.sh.j2 | 5 +++-- .../database_backup/database_backup-mysql/defaults/main.yml | 1 + .../database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- .../sync/database_sync/database_sync-mysql/defaults/main.yml | 1 + roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 10 insertions(+), 5 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 06470a09..9821f131 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,6 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" +_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index c6c57269..29b8003b 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,3 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index 901cfcf7..fb60cdea 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,8 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - mysqldump -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" + {{ cron_mysql_backup.mysqldump_command }} \ + -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } @@ -23,4 +24,4 @@ cleanup(){ } backup -cleanup \ No newline at end of file +cleanup diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 3ad97b6c..09f44ca3 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,6 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 4b470b89..57a46e2e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index fe32089a..cbade053 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 48e3e4a0..008f0cf4 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,5 +1,6 @@ --- mysql_sync: + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index b8182bc3..778092bc 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From a9cc7adf64a8c7959f971df84df9e6172d0cf4c6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 18:18:47 +0200 Subject: [PATCH 07/29] Fix database backups pr 1.x (#109) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. * Defaults file must be the first param for mysqldump. --- roles/_init/defaults/main.yml | 2 +- .../cron_database_backup-mysql/defaults/main.yml | 2 +- .../cron_database_backup-mysql/templates/regular-backups.sh.j2 | 2 +- roles/database_backup/database_backup-mysql/defaults/main.yml | 2 +- .../database_backup/database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- roles/sync/database_sync/database_sync-mysql/defaults/main.yml | 2 +- roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9821f131..c4d15b78 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,7 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" -_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index 29b8003b..f0c869f3 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,4 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index fb60cdea..33c27a5a 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,7 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - {{ cron_mysql_backup.mysqldump_command }} \ + mysqldump {{ cron_mysql_backup.mysqldump_params }} \ -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 09f44ca3..f7e070e9 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,7 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 57a46e2e..d06b1543 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index cbade053..1db841b0 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 008f0cf4..0ab99e98 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,6 @@ --- mysql_sync: - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 778092bc..c3ff0b9e 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From 4c4c3ad1f9ecd25e25e9c2e5f79d4cb8108c9f34 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 22 Apr 2022 10:28:11 +0100 Subject: [PATCH 08/29] Fix MySQL backup deferral. (#110) --- .../cron_database_backup-mysql/tasks/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index 40b0c47e..d09fbbac 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -47,4 +47,4 @@ minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" - delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" + delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}" From 029848d1163191087bd85f372973594b283aad17 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 26 Apr 2022 11:30:31 +0100 Subject: [PATCH 09/29] Files recurse fix pr 1.x (#112) * Don't recurse through site directory when setting permissions during config_generate step. * Update drupal7 config_generate perms update task and use true/false instead of yes/no. --- .../config_generate/config_generate-drupal7/tasks/settings.yml | 3 +-- .../config_generate/config_generate-drupal8/tasks/settings.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f82c9aa7..277899a0 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -15,9 +15,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index a9937d8e..70d9b113 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -19,9 +19,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: From b71fa83d6b4deda1daa4532c3a00e3835118cf4a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 29 Apr 2022 11:52:28 +0100 Subject: [PATCH 10/29] Improve multisite support (#115) * Pass -l option in drush commands to specify site name, which should match the site folder name. * Move -l option in drush commands to the first option instead of the last. --- roles/cache_clear/cache_clear-drupal7/tasks/main.yml | 2 +- roles/cache_clear/cache_clear-drupal8/tasks/main.yml | 2 +- .../database_apply-drupal7/tasks/features.yml | 4 ++-- roles/database_apply/database_apply-drupal7/tasks/main.yml | 4 ++-- roles/database_apply/database_apply-drupal8/tasks/main.yml | 6 +++--- .../maintenance_mode-drupal-core/tasks/offline.yml | 4 ++-- .../maintenance_mode-drupal-core/tasks/online.yml | 4 ++-- .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 4 ++-- .../admin_creds/admin_creds-drupal8/tasks/admin.yml | 4 ++-- roles/sanitize/sanitize-drupal8/tasks/main.yml | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml index 1261ff4b..6f72b3a5 100644 --- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal 7 cache. command: - cmd: "{{ drush_bin }} -y cc all" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index cfd7fdd3..5b780086 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal cache. command: - cmd: "{{ drush_bin }} -y cr" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index 19d58e9b..edba46bf 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,8 +1,8 @@ --- - name: Check if Features module is enabled. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} pm-info features | grep ': enabled' | wc -l" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" register: features_enabled - name: Revert Drupal configuration from Features. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -y {{ site.revert_features_command }}" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}" when: features_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 048a1457..7c2201e2 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -21,7 +21,7 @@ - name: Apply Drupal database updates. shell: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" loop_control: diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 5634faca..dc138585 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -15,7 +15,7 @@ - name: Install Drupal. command: - cmd: "{{ drush_bin }} {{ site.install_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -34,7 +34,7 @@ - name: Apply Drupal database updates. command: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -44,7 +44,7 @@ - name: Import configuration. command: - cmd: "{{ drush_bin }} -y {{ site.config_import_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 2872226a..cbe65b07 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -2,7 +2,7 @@ # @todo this needs refactoring for multisite. - name: Enable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 1 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -12,7 +12,7 @@ - name: Enable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 1" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 99057084..e6a9cbf5 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,7 +1,7 @@ --- - name: Disable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 0 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -10,7 +10,7 @@ - name: Disable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 0" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml index 98401c5c..c8d75bd7 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml @@ -6,6 +6,6 @@ _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}" - name: Reset admin username. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" - name: Reset admin password. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml index 0302308b..c2adafdf 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml @@ -4,14 +4,14 @@ # Loading the user directly is akward, but at least means we don't bypass entity update. - name: Reset admin username. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.username | length > 1 - name: Reset admin password. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.password | length > 1 diff --git a/roles/sanitize/sanitize-drupal8/tasks/main.yml b/roles/sanitize/sanitize-drupal8/tasks/main.yml index 0a33f5a8..26675f27 100644 --- a/roles/sanitize/sanitize-drupal8/tasks/main.yml +++ b/roles/sanitize/sanitize-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Sanitize database. command: - cmd: "{{ drush_bin }} -y {{ site.sanitize_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.sanitize_command }}" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" From c933dc8224e42106df909e29cdb8928f5c5da580 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 23 May 2022 16:19:31 +0100 Subject: [PATCH 11/29] Static credentials handling fix pr 1.x (#119) * Attempt to fix static credentials handling when deploying to an ASG. * Include build_type in static password file location. * Move to using new location for static credentials handling, but try to catch any legacy stuff too, for now. * Add a couple of comments to legacy handling of static credentials. --- .../database_backup-mysql/tasks/deploy.yml | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index a67d240e..dbb39de7 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -24,8 +24,55 @@ - set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' + +### Start of legacy handling. Can remove in a few months. +- name: Catch legacy static password handling. + ansible.builtin.stat: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + register: _legacy_static_creds + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Create new static password location if it doesn't exist. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0755 + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Copy legacy static password to new location. + ansible.builtin.copy: + src: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + dest: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0644 + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists + +- name: Delete legacy static password file if it exists. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + state: absent + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists +### End of legacy handling. + - set_fact: - _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" + _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. - set_fact: From b6ea0206e8f83f7e2459babd0622403b955276af Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 31 May 2022 16:26:10 +0200 Subject: [PATCH 12/29] Making contents of deploy tar 'ownerless'. (#117) --- roles/deploy_code/tasks/cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 4e648d25..03530f22 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -22,7 +22,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 1c2b206d009b21d10b96c434168c15ac5700b96c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 7 Jun 2022 15:48:04 +0100 Subject: [PATCH 13/29] Implement file syncing (#124) * Add a files_sync role. * Clear up some comments, add temp_dir variable and use rsync with command module instead of synchronize module, in files_sync role. * Need trailing slash on src when syncing files to destination server so the contents get synced and not the parent directory itself. * Use rsync instead of copy when syncing files to target server as copy is so slow. * Ensure file sync tasks are run only once. --- roles/sync/files_sync/defaults/main.yml | 16 ++++++++++++++++ roles/sync/files_sync/tasks/main.yml | 7 +++++++ roles/sync/files_sync/tasks/sync.yml | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 roles/sync/files_sync/defaults/main.yml create mode 100644 roles/sync/files_sync/tasks/main.yml create mode 100644 roles/sync/files_sync/tasks/sync.yml diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml new file mode 100644 index 00000000..5b65b1fb --- /dev/null +++ b/roles/sync/files_sync/defaults/main.yml @@ -0,0 +1,16 @@ +--- +files_sync: + directories: + - source: + # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files" + # Host that contains source files. + host: "localhost" + # Location on deploy server where source files get copied to first. NO TRAILING SLASH. + temp_dir: "/tmp" + # Used to create directory in /tmp. + build_id: mybuildprod + target: + # Location of the files to sync to. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files" + build_id: mybuilddev diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml new file mode 100644 index 00000000..47c3667a --- /dev/null +++ b/roles/sync/files_sync/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Sync files. + include_tasks: "sync.yml" + with_items: "{{ files_sync.directories }}" + loop_control: + loop_var: files + run_once: true diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml new file mode 100644 index 00000000..e2160008 --- /dev/null +++ b/roles/sync/files_sync/tasks/sync.yml @@ -0,0 +1,21 @@ +--- +- name: Create a temporary directory for source files on localhost. + ansible.builtin.file: + path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + delegate_to: localhost + run_once: true + +- name: Copy the source files onto the deploy server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" + delegate_to: "localhost" + run_once: true + +- name: Copy the source files from the deploy server onto the destination server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: "localhost" + run_once: true From 1cd0ed0797c1d26a933b4a3e06e0279e4af6a05a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Wed, 8 Jun 2022 12:47:34 +0100 Subject: [PATCH 14/29] Create Drupal-specific sync roles (#128) * Add a drupal_sync_tasks role to run Drupal specific tasks during syncs. * Move sync database_apply files into tasks subdirectory. * Woops, used bad role names in sync database update roles. * Remove feature reverting from drupal7 database_apply sync role. --- .../cache_clear-drupal7/tasks/main.yml | 11 ++++++++ .../cache_clear-drupal8/tasks/main.yml | 11 ++++++++ .../database_apply-drupal7/tasks/main.yml | 12 +++++++++ .../database_apply-drupal8/tasks/main.yml | 26 +++++++++++++++++++ .../database_apply/tasks/main.yml | 6 +++++ 5 files changed, 66 insertions(+) create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml new file mode 100644 index 00000000..ab9342ba --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal 7 cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml new file mode 100644 index 00000000..96c721ee --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml new file mode 100644 index 00000000..f520514e --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.shell: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml new file mode 100644 index 00000000..bb9cc73c --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Import configuration. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - site.config_import_command + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml new file mode 100644 index 00000000..58f6215a --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Apply database updates. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/database_apply/database_apply-{{ project_type }}" + when: deploy_operation == 'deploy' + run_once: true From 904033f8bf3e1973d7b78c8ea2072426f1d0decb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 10 Jun 2022 12:57:40 +0200 Subject: [PATCH 15/29] Fixing GRANT query for MySQL > 8.0. (#131) * Fixing GRANT query for MySQL > 8.0. * Create what though? --- roles/_init/defaults/main.yml | 1 + roles/database_backup/database_backup-mysql/tasks/deploy.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index c4d15b78..790e9409 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -1,6 +1,7 @@ --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index dbb39de7..b0328958 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -88,8 +88,9 @@ # to allow this role to be looped over, # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. +# As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 94715d8bd5363c640649e09bffe0e3b1574ceb90 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 13 Jun 2022 10:34:12 +0100 Subject: [PATCH 16/29] Use IF NOT EXISTS when creating database user as that command fails if static database users are being used. (#133) --- roles/database_backup/database_backup-mysql/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index b0328958..f5a7130a 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -90,7 +90,7 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 42f12c81d0974648857d203139a154c736ac9731 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 16 Jun 2022 13:39:30 +0100 Subject: [PATCH 17/29] Attempt to fix syncs whenever the 'dump' type is used for source or target. (#149) --- .../database_sync-mysql/tasks/sync.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c3ff0b9e..98eca0f3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -38,12 +38,6 @@ - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" - when: not database.source.type == 'dump' - -- name: Register existing remote dump name. - set_fact: - mysql_sync_source_dump_path: "{{ database.source.database }}" - when: database.source.type == 'dump' - name: Get source last known good build number. command: @@ -70,7 +64,6 @@ shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - - not database.source.type == 'dump' - database.source.fresh_db - name: Find source database host. @@ -97,12 +90,6 @@ - name: Register tmp target dump name. set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" - when: not database.target.type == 'dump' - -- name: Register final target dump name. - set_fact: - mysql_sync_target_dump_path: "{{ database.target.database }}" - when: database.target.type == 'dump' - name: Get target last known good build number. command: @@ -139,21 +126,17 @@ - name: Drop target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Recreate target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Repopulate database from dump. shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - when: not database.target.type == 'dump' - name: Remove tmp dump file. file: path: "{{ mysql_sync_target_dump_path }}" state: absent - when: not database.target.type == 'dump' - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > From a9c6abfb5c686d030a0671d8a20805b7d3b1be02 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 20 Jun 2022 11:31:01 +0200 Subject: [PATCH 18/29] Squashfs pr 1.x (#150) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. --- roles/_init/tasks/main.yml | 51 +++++++++---- roles/deploy_code/defaults/main.yml | 13 +++- roles/deploy_code/tasks/cleanup.yml | 108 +++++++++++++++++++++++++++- roles/live_symlink/tasks/main.yml | 23 ++++-- 4 files changed, 171 insertions(+), 24 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a1b6e204..53be27d3 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,27 +1,45 @@ --- # Ensure default values for common variables. - name: Define deploy user. - set_fact: + ansible.builtin.set_fact: deploy_user: "{{ deploy_user | default('deploy') }}" - name: Define deploy base path. - set_fact: + ansible.builtin.set_fact: deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}" - name: Define mounted directory for assets. - set_fact: + ansible.builtin.set_fact: deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}" - name: Define webroot. - set_fact: + ansible.builtin.set_fact: webroot: "{{ webroot | default('web') }}" - name: Define build deploy path prefix. - set_fact: + ansible.builtin.set_fact: deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_" - name: Define build deploy path. - set_fact: + ansible.builtin.set_fact: deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}" - name: Define live_symlink dest. - set_fact: + ansible.builtin.set_fact: live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" +# Manipulate variables for SquashFS builds. +- name: Define image builds base path. + ansible.builtin.set_fact: + build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" + when: deploy_code.mount_type == "squashfs" +- name: Define image builds build path prefix. + ansible.builtin.set_fact: + build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" + when: deploy_code.mount_type == "squashfs" +- name: Define live_symlink dest for image builds. + ansible.builtin.set_fact: + live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" + when: deploy_code.mount_type == "squashfs" +- name: Overwrite deploy and live_symlink paths if SquashFS deploy. + ansible.builtin.set_fact: + deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" + when: deploy_code.mount_type == "squashfs" + # Gather last known good build directly from symlink. # This can happen: # - when the first builds failed, @@ -42,7 +60,7 @@ # This is passed from caller. - name: Gather last known good build number. - set_fact: + ansible.builtin.set_fact: previous_build_number: "{{ previous_known_build_number }}" # - set_fact: @@ -53,27 +71,34 @@ # Make sure the deploy target exists. - name: Ensure deploy target directory exists. - file: + ansible.builtin.file: path: "{{ deploy_base_path }}" state: directory +# Make sure the build target exists. +- name: Ensure build target directory exists. + ansible.builtin.file: + path: "{{ build_base_path }}" + state: directory + when: deploy_code.mount_type == "squashfs" + # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. - stat: + ansible.builtin.stat: path: "{{ role_path }}/tasks/{{ project_type }}.yml" register: _project_type_task_result delegate_to: "localhost" # Project specific init tasks. - name: Include project init tasks. - include_tasks: "{{ project_type }}.yml" + ansible.builtin.include_tasks: "{{ project_type }}.yml" when: - _project_type_task_result.stat.exists - name: Define opcache cachetool path. - set_fact: + ansible.builtin.set_fact: cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}" - name: Ensure we have a cachetool binary. - import_role: + ansible.builtin.import_role: name: cli/cachetool diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index d616bf08..f1f7cdb4 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -3,7 +3,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -15,12 +15,21 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false # Details of API call to trigger. See api_call role. diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 03530f22..c722b049 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -6,19 +6,39 @@ become: true when: "www_user != deploy_user" -- name: Ensure permissions are set on directory. +- name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type != "squashfs" -- name: Delete codebases. +- name: Ensure permissions are set on builds directory. + ansible.builtin.shell: + cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.perms_fix_path is defined + - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Delete codebases from deploy directory. ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type != "squashfs" + +- name: Delete codebases from builds directory. + ansible.builtin.file: + name: "{{ build_path_prefix }}{{ item }}" + state: absent + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type == "squashfs" - name: Create a tarball of the deployed codebases. ansible.builtin.command: @@ -26,6 +46,26 @@ when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Ensure older SquashFS images are deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + run_once: true + +- name: Create a SquashFS image of the deployed codebases. + ansible.builtin.command: + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true - name: Create destination folder. @@ -38,14 +78,76 @@ - deploy_code.mount_sync | length > 1 run_once: true -- name: Move to final destination. +- name: Move tar file to final destination. ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Move SquashFS image to final destination. + ansible.builtin.command: + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true +- name: Copy SquashFS image to local server. + ansible.builtin.command: + cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Check if we have a mount already. + ansible.builtin.shell: + cmd: "mount | grep {{ deploy_base_path }}" + ignore_errors: true + register: _mount_check + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount -l {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + +- name: Reload any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: reloaded + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + - deploy_code.services | length > 0 + +- name: Mount new SquashFS image. + ansible.builtin.command: + cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: name: api_call diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml index 460ced9a..fa1d9e97 100644 --- a/roles/live_symlink/tasks/main.yml +++ b/roles/live_symlink/tasks/main.yml @@ -1,19 +1,28 @@ --- -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ previous_build_number }}" when: deploy_operation == 'revert' +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_dest }}" + +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_build_dest }}" + when: deploy_code.mount_type == "squashfs" + - name: Symlink build. - file: + ansible.builtin.file: src: "{{ _live_symlink_build_target }}" - dest: "{{ live_symlink_dest }}" + dest: "{{ _live_symlink_dest_target }}" state: link + follow: false + force: true - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ live_symlink.templates }}" @@ -24,10 +33,12 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link + follow: false + force: true with_items: "{{ live_symlink.symlinks }}" loop_control: loop_var: link From a4de7710d2d7609aa9213c1f73951eaf27b23cbd Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 24 Jun 2022 13:00:24 +0100 Subject: [PATCH 19/29] Check deploy_code.mount_type is defined when setting facts in init role. (#155) --- roles/_init/tasks/main.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 53be27d3..effa2790 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -26,19 +26,27 @@ - name: Define image builds base path. ansible.builtin.set_fact: build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define image builds build path prefix. ansible.builtin.set_fact: build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define live_symlink dest for image builds. ansible.builtin.set_fact: live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Overwrite deploy and live_symlink paths if SquashFS deploy. ansible.builtin.set_fact: deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Gather last known good build directly from symlink. # This can happen: @@ -80,7 +88,9 @@ ansible.builtin.file: path: "{{ build_base_path }}" state: directory - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. From fe2aaf10f5bfc9124f5c1ab1cb6bff6e8e5b635c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 28 Jun 2022 11:08:07 +0100 Subject: [PATCH 20/29] Make config imports during syncs optional (#157) * Use a different variable for config imports during a sync so they can be optional. * Actually, no. Use a variable to determine if config should be imported during a sync. * Fix some logic with config import during syncs and add a comment to explain when the task runs. --- .../database_apply/database_apply-drupal8/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index bb9cc73c..e050af3f 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -9,6 +9,7 @@ loop_control: loop_var: site +# This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" @@ -20,6 +21,7 @@ loop_var: site when: - site.config_import_command + - site.sync_config_import is not defined or site.sync_config_import - name: Clear the cache. ansible.builtin.include_role: From 451227d28461c0afcc0bb854277fda17796d1d9e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jun 2022 17:40:45 +0200 Subject: [PATCH 21/29] Squashfs pr 1.x (#153) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. * Adding a revert behaviour for SquashFS. * Making some wording a little less ambiguous. * Fixing bad image path and adding clauses to stat check. * Trying to make remounting SquashFS images a bit safer. * Adding when clauses to SquashFS image path check. * Removing revert code, as it cannot work. --- roles/deploy_code/tasks/cleanup.yml | 64 +++++++++++++++++++---------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index c722b049..e1412749 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -49,16 +49,6 @@ - deploy_code.mount_type == "tarball" run_once: true -- name: Ensure older SquashFS images are deleted. - ansible.builtin.file: - path: "{{ build_base_path }}/deploy.sqsh" - state: absent - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - run_once: true - - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" @@ -96,6 +86,36 @@ - deploy_code.mount_type == "squashfs" run_once: true +- name: Check if we have a SquashFS image already there. + ansible.builtin.stat: + path: "{{ build_base_path }}/deploy.sqsh" + register: _deploy_code_mount_image + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Copy previous SquashFS image in case of rollback. + ansible.builtin.copy: + remote_src: true + force: true + src: "{{ build_base_path }}/deploy.sqsh" + dest: "{{ build_base_path }}/deploy_previous.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_image.stat.islnk is defined + +- name: Ensure mounted SquashFS image is deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Copy SquashFS image to local server. ansible.builtin.command: cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" @@ -108,21 +128,11 @@ ansible.builtin.shell: cmd: "mount | grep {{ deploy_base_path }}" ignore_errors: true - register: _mount_check - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Unmount existing SquashFS image. - ansible.builtin.command: - cmd: "umount -l {{ deploy_base_path }}" - become: true + register: _deploy_code_mount_check when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded - name: Reload any services that might be keeping the loop device busy. ansible.builtin.service: @@ -136,9 +146,19 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded + - _deploy_code_mount_check is succeeded - deploy_code.services | length > 0 +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - name: Mount new SquashFS image. ansible.builtin.command: cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" From c7aae71b8903846fccbbd325464509282cb6a302 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 16 Aug 2022 12:18:58 +0100 Subject: [PATCH 22/29] Add cache clears to Drupal deployments, before DB updates and stuff (#159) * Add cache clears to Drupal deployments, before database updates and config import are run. * Move cache clears out of meta roles and only run in database_apply role on non-initial builds. * Add cache clear stuff to drupal7 database_apply role. --- roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++ roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 7c2201e2..8b655bda 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -19,6 +19,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index dc138585..23cb0b3b 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -32,6 +32,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 15b8daae0b9736f2a80878415a02155dd2acc8e0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 5 Sep 2022 18:05:10 +0200 Subject: [PATCH 23/29] Avoid leaving exponentially growing sqsh files in build locations! (#164) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. --- roles/deploy_code/tasks/cleanup.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index e1412749..b206b7ab 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -95,17 +95,18 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" -- name: Copy previous SquashFS image in case of rollback. +- name: Copy previous SquashFS image to shared mount point in case of rollback. ansible.builtin.copy: remote_src: true force: true src: "{{ build_base_path }}/deploy.sqsh" - dest: "{{ build_base_path }}/deploy_previous.sqsh" + dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - _deploy_code_mount_image.stat.islnk is defined + run_once: true - name: Ensure mounted SquashFS image is deleted. ansible.builtin.file: From 5cab4ae00d301e183ab1d17cf53bf289391b4a6b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 6 Oct 2022 16:28:30 +0200 Subject: [PATCH 24/29] Exclude sqsh file pr 1.x (#167) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. * Making sure image/tarball filenames are build specific in /tmp. --- roles/deploy_code/tasks/cleanup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index b206b7ab..353854b3 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -42,7 +42,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -70,7 +70,7 @@ - name: Move tar file to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -79,7 +79,7 @@ - name: Move SquashFS image to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 459faba5e163ee6355d4bb56f9ad19042c7aa148 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 7 Oct 2022 16:05:35 +0200 Subject: [PATCH 25/29] Removing unnecessary lines in Drupal config generation. (#169) * Removing unnecessary lines in Drupal config generation. * Drupal 7 lacked the install_command var. --- .../tasks/settings.yml | 16 ++++++--------- .../tasks/settings.yml | 20 ++++++++----------- .../database_apply-drupal7/tasks/main.yml | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index 277899a0..f8e3b55e 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -1,25 +1,21 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set private files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index 70d9b113..ebf182e8 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -1,29 +1,25 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" -- name: set private files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set config directory path. - set_fact: +- name: Set config directory path. + ansible.builtin.set_fact: build_config_sync_directory: "{{ deploy_path }}/{{ site.config_sync_directory }}" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 8b655bda..eb2edbfa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -45,4 +45,4 @@ include_tasks: ctools.yml with_items: "{{ drupal.sites }}" loop_control: - loop_var: site \ No newline at end of file + loop_var: site From 200f59002c0459d63edc4d821f7b31ee94dde351 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 10 Oct 2022 14:31:48 +0200 Subject: [PATCH 26/29] Ensuring dump directory exists on backup step. (#172) * Ensuring dump directory exists on backup step. * Acts on shared storage, so can run once. --- .../database_backup-mysql/tasks/deploy-dump.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index d06b1543..80a6e5ef 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" @@ -14,7 +14,15 @@ when: previous_build_number == 0 run_once: true +- name: Ensure the dump directory exists. + ansible.builtin.file: + path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + run_once: true + - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true From 2c7443277b94486bad5f3d7b57212c9426bf5e9f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 13:49:41 +0200 Subject: [PATCH 27/29] Allowing Drupal 7 jobs to disable cron. (#174) --- roles/cron/cron_drupal7/defaults/main.yml | 1 + roles/cron/cron_drupal7/tasks/cron.yml | 2 +- roles/cron/cron_drupal7/tasks/job.yml | 10 ++++++---- roles/cron/cron_drupal7/tasks/main.yml | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index a09c1036..a6551ab6 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -8,6 +8,7 @@ drupal: - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] diff --git a/roles/cron/cron_drupal7/tasks/cron.yml b/roles/cron/cron_drupal7/tasks/cron.yml index 2bc2c6c7..056d50e0 100644 --- a/roles/cron/cron_drupal7/tasks/cron.yml +++ b/roles/cron/cron_drupal7/tasks/cron.yml @@ -1,6 +1,6 @@ --- - name: Setup Drupal 7 cron tasks. - include_tasks: + ansible.builtin.include_tasks: file: job.yml with_items: "{{ site.cron }}" loop_control: diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index 841c4b51..0f1d3203 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -1,10 +1,10 @@ --- - name: Define cron job command. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\"" when: - drupal.defer is defined @@ -13,7 +13,7 @@ - drupal.defer_target | length > 0 - name: Define cron job command for differing deploy users. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'" when: - www_user != deploy_user @@ -21,9 +21,11 @@ - drupal.defer - name: Setup Drupal cron tasks on app server. - cron: + ansible.builtin.cron: name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" job: "{{ _cron_job_command }}" + state: present + disabled: "{{ entry.disabled | default(omit) }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index 75fa05c7..cf0f80f6 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -10,7 +10,7 @@ - drupal.defer - name: Create site cron entries. - include_tasks: + ansible.builtin.include_tasks: file: cron.yml with_items: "{{ drupal.sites }}" loop_control: From 03852117a42d9aa62e30d3161ae0b418d6e440bf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 17:41:52 +0200 Subject: [PATCH 28/29] Suppress db revert pr 1.x (#177) * Supporting option to suppress reverting backups. * Fixing namespacing in database_backup role. --- .../tasks/cleanup-dump.yml | 2 +- .../tasks/cleanup-rolling.yml | 2 +- .../database_backup-mysql/tasks/cleanup.yml | 12 ++++---- .../tasks/deploy-none.yml | 6 ++-- .../tasks/deploy-rolling.yml | 8 +++--- .../database_backup-mysql/tasks/deploy.yml | 28 +++++++++---------- .../database_backup-mysql/tasks/main.yml | 2 +- .../tasks/revert-dump.yml | 2 +- .../database_backup-mysql/tasks/revert.yml | 14 ++++++---- roles/database_backup/defaults/main.yml | 1 + roles/database_backup/tasks/main.yml | 4 +-- 11 files changed, 42 insertions(+), 39 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index d5774a78..a9ea9ae2 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql dumps. - file: + ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index b131f8e6..2bdf29ee 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql databases. - mysql_db: + community.mysql.mysql_db: name: "{{ database.database }}_{{ item }}" state: absent login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index 71570baa..ac71fcbb 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -1,23 +1,23 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" # We assume it's safe to only go back 50 build back. - name: Delete mysql users. - mysql_user: + community.mysql.mysql_user: name: "{{ database.user }}_{{ item }}" host_all: true state: absent diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml index 65c68978..dea8fe9f 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 1db841b0..71fa7580 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -1,17 +1,17 @@ --- # Setup database names. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}" # Note: we don't use the mysql_db Ansible module on purpose. # If database already exists, we want to fail and not override it # with previous build. - name: Create new database. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index f5a7130a..fc06be74 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -1,27 +1,27 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" # Credentials for the app itself. # Manual: reuse given creds. Note the user gets ignored. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ _mysql_user }}" when: mysql_backup.credentials_handling == 'manual' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ _mysql_password }}" when: mysql_backup.credentials_handling == 'manual' # Static: create/update user/pwd only once. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' @@ -71,18 +71,18 @@ - _legacy_static_creds.stat.exists ### End of legacy handling. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}_{{ build_number }}" when: mysql_backup.credentials_handling == 'rotate' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}" when: mysql_backup.credentials_handling == 'rotate' -- include_tasks: "deploy-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml" # We append privileges instead of replacing, # to allow this role to be looped over, @@ -90,11 +90,11 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true -- set_fact: +- ansible.builtin.set_fact: _build_database: original: "{{ database }}" name: "{{ _mysql_build_database_name }}" @@ -102,5 +102,5 @@ password: "{{ _mysql_build_password }}" host: "{{ _mysql_host }}" -- set_fact: +- ansible.builtin.set_fact: build_databases: "{{ build_databases + [ _build_database ] }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/main.yml b/roles/database_backup/database_backup-mysql/tasks/main.yml index ec10df75..b015ac09 100644 --- a/roles/database_backup/database_backup-mysql/tasks/main.yml +++ b/roles/database_backup/database_backup-mysql/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Take database backup. - include_tasks: "{{ deploy_operation }}.yml" + ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" with_items: "{{ mysql_backup.databases }}" loop_control: loop_var: database diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 735a9805..f914ee78 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,5 @@ --- - name: Revert database from dump. - shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml index ba054b44..f6076262 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml @@ -1,17 +1,19 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "revert-{{ mysql_backup.handling }}.yml" - when: previous_build_number > 0 +- ansible.builtin.include_tasks: "revert-{{ mysql_backup.handling }}.yml" + when: + - previous_build_number > 0 + - database_backup.revert diff --git a/roles/database_backup/defaults/main.yml b/roles/database_backup/defaults/main.yml index 9201fa20..df4881c0 100644 --- a/roles/database_backup/defaults/main.yml +++ b/roles/database_backup/defaults/main.yml @@ -2,3 +2,4 @@ database_backup: engines: - mysql + revert: true \ No newline at end of file diff --git a/roles/database_backup/tasks/main.yml b/roles/database_backup/tasks/main.yml index 83adedf9..829536c2 100644 --- a/roles/database_backup/tasks/main.yml +++ b/roles/database_backup/tasks/main.yml @@ -1,9 +1,9 @@ --- -- set_fact: +- ansible.builtin.set_fact: build_databases: [] - name: Take database dumps. - include_role: + ansible.builtin.include_role: name: "database_backup/database_backup-{{ engine }}" with_items: "{{ database_backup.engines }}" loop_control: From 072ff15cc74a539d69f27b9dddaa926753550aef Mon Sep 17 00:00:00 2001 From: gregharvey Date: Fri, 18 Nov 2022 13:55:18 +0100 Subject: [PATCH 29/29] Fixing bad assumption that databases will have TCP connections. --- .../database_backup/database_backup-mysql/tasks/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index fc06be74..5442230d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -89,11 +89,16 @@ # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. -- name: Create/update mysql user. +- name: Create/update mysql user for TCP connections. ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true +- name: Create/update mysql user for unix socket connections. + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';" + when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + run_once: true + - ansible.builtin.set_fact: _build_database: original: "{{ database }}"