From 3ccfe51727c35f7cef25216f0ae4d0701d9dabb9 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 20 Jan 2022 10:13:40 +0000 Subject: [PATCH 01/40] Defer nightly backups, disable ASG processes during syncs and run syncs with backup (#94) * Change location of nightly backup script and delegate the cron that runs it to the deploy server, if required (for ASGs). * Set up nightly backup crons in separate files in /etc/cron.d * Can't put site cron files in /etc/cron.d because the deploy user doesn't have perms. * Try and add the ability to sync a site using a nightly backup instead of a fresh DB dump. * Use Ansible modules to look up RDS host and to copy the nightly backup into place. * Delegate PATH setup in db backup cron to localhost. * Shell bad. Command good. But makes it convoluted. Oh well. * Used wrong database name in source database copy. * Try and disable the ReplaceUnhealthy auto scale process during syncs. * Fix Drupal cron roles when deferring to deploy server. --- .../cron_database_backup-mysql/tasks/main.yml | 10 +++ .../tasks/setup.yml | 18 ++++- roles/cron/cron_drupal7/tasks/main.yml | 1 + roles/cron/cron_drupal8/tasks/main.yml | 1 + .../database_sync-mysql/defaults/main.yml | 9 +++ .../database_sync-mysql/tasks/sync.yml | 79 ++++++++++++++++++- 6 files changed, 115 insertions(+), 3 deletions(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml index 04789df8..ffd14157 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/main.yml @@ -3,6 +3,16 @@ # screwing the daily backup when using rolling db, we could # add a keep mechanism for backup scripts, like for the dumps themselves. # Nice to have more than anything. +- name: Setup PATH in crontab. + cron: + name: PATH + env: true + job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost + when: + - drupal.defer is defined + - drupal.defer + - include_tasks: setup.yml vars: database: database diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index bdc1534c..40b0c47e 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -25,12 +25,26 @@ - name: Create backup script. template: src: "regular-backups.sh.j2" - dest: "/home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + dest: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" mode: 0700 +- name: Define backup cron job command. + set_fact: + _backup_cron_job_command: "/bin/sh /home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + +- name: Define backup cron job command if deferred (ASG). + set_fact: + _backup_cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _backup_cron_job_command }}\"" + when: + - drupal.defer is defined + - drupal.defer + - drupal.defer_target is defined + - drupal.defer_target | length > 0 + - name: Setup regular backup for MySQL. cron: name: "cron_mysql_{{ database.host }}_{{ database.original.database }}" minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" - job: "/bin/sh /home/{{ deploy_user }}/{{ database.host }}-{{ database.original.database }}-regular-backups.sh" + job: "{{ _backup_cron_job_command }}" + delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index d0cbfce9..75fa05c7 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -4,6 +4,7 @@ name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" + delegate_to: localhost when: - drupal.defer is defined - drupal.defer diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index c1d7b259..48e3e4a0 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -15,6 +15,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -25,3 +32,5 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 81e9b2ce..b8182bc3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -1,4 +1,40 @@ --- +- name: Get database source host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_source_database_host_info + delegate_to: "{{ database.source.host }}" + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +# This task does not need a delegate_to because the hosts set in the sync playbook in the repo should be the target host. +- name: Get database target host region. + amazon.aws.ec2_metadata_facts: + register: mysql_sync_target_database_host_info + when: + - database.target.asg is defined + - database.target.asg | length > 0 + +- name: Disable ReplaceUnhealthy autoscale process on source ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.source.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Disable ReplaceUnhealthy autoscale process on target ASG. + ansible.builtin.command: > + aws autoscaling suspend-processes --auto-scaling-group-name {{ database.target.asg }} --scaling-processes ReplaceUnhealthy --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 + - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" @@ -33,7 +69,30 @@ - name: Take a dump from source database. shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" - when: not database.source.type == 'dump' + when: + - not database.source.type == 'dump' + - database.source.fresh_db + +- name: Find source database host. + ansible.builtin.command: + cmd: "grep 'host' {{ database.source.credentials_file }}" + register: mysql_host_info_grep + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Register source database host. + set_fact: + mysql_sync_source_database_host: "{{ mysql_host_info_grep.stdout.split('=')[1] }}" + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db + +- name: Copy a nightly backup for the source database. + ansible.builtin.copy: + src: "{{ database.source.dumps_directory }}/{{ mysql_sync_source_database_host }}/{{ database.source.database }}" + dest: "{{ mysql_sync_source_dump_path }}" + remote_src: true + delegate_to: "{{ database.source.host }}" + when: not database.source.fresh_db - name: Register tmp target dump name. set_fact: @@ -95,3 +154,21 @@ path: "{{ mysql_sync_target_dump_path }}" state: absent when: not database.target.type == 'dump' + +- name: Enable all autoscale processes on source ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.source.asg }} --region {{ mysql_sync_source_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.source.asg is defined + - database.source.asg | length > 0 + - database.source.fresh_db is defined + - database.source.fresh_db + +- name: Enable all autoscale processes on target ASG. + ansible.builtin.command: > + aws autoscaling resume-processes --auto-scaling-group-name {{ database.target.asg }} --region {{ mysql_sync_target_database_host_info.ansible_facts.ansible_ec2_instance_identity_document_region }} + delegate_to: localhost + when: + - database.target.asg is defined + - database.target.asg | length > 0 From a54c09cb0ce5b6889e672ba8ea741b71a3da034b Mon Sep 17 00:00:00 2001 From: Dionisio Date: Thu, 10 Mar 2022 14:16:13 +0100 Subject: [PATCH 02/40] Added deploy.yml examples for Drupal 9 and Localgov. Updated tests (#97) --- .github/workflows/ce-deploy-test.yml | 22 ++++++++ ce-dev/ansible/examples/drupal9/deploy.yml | 55 +++++++++++++++++++ ce-dev/ansible/examples/localgov/deploy.yml | 59 +++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 ce-dev/ansible/examples/drupal9/deploy.yml create mode 100644 ce-dev/ansible/examples/localgov/deploy.yml diff --git a/.github/workflows/ce-deploy-test.yml b/.github/workflows/ce-deploy-test.yml index baa13347..141fc8e7 100644 --- a/.github/workflows/ce-deploy-test.yml +++ b/.github/workflows/ce-deploy-test.yml @@ -59,6 +59,28 @@ jobs: curl https://www.test.local shell: bash + - name: Run a test ce-dev deploy with Drupal 9 + run: | + ce-dev create -p testnine -t drupal9 -d ~/testnine + cd ~/testnine + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testnine.local + shell: bash + + - name: Run a test ce-dev deploy with Localgov + run: | + ce-dev create -p testlocalgov -t localgov -d ~/testlocalgov + cd ~/testlocalgov + ce-dev init + ce-dev start + ce-dev provision + ce-dev deploy + curl https://www.testlocalgov.local + shell: bash + # Builds the table of contents for the docs - name: Documentation (build table of contents) if: ${{ github.event.pull_request.base.ref == '1.x' }} diff --git a/ce-dev/ansible/examples/drupal9/deploy.yml b/ce-dev/ansible/examples/drupal9/deploy.yml new file mode 100644 index 00000000..deeef9ee --- /dev/null +++ b/ce-dev/ansible/examples/drupal9/deploy.yml @@ -0,0 +1,55 @@ +--- +# Template playbook for a local Drupal9 codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: no + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file diff --git a/ce-dev/ansible/examples/localgov/deploy.yml b/ce-dev/ansible/examples/localgov/deploy.yml new file mode 100644 index 00000000..9844206b --- /dev/null +++ b/ce-dev/ansible/examples/localgov/deploy.yml @@ -0,0 +1,59 @@ +--- +# Template playbook for a local localgov codebase. +- hosts: deploy-web + vars: + - project_name: example + - project_type: drupal8 + - webroot: web + - build_type: local + - _env_type: dev + - _domain_name: www.{{ project_name }}.local + # Path to your project root. This must match the "volume" set in the docker-compose template. + - deploy_path: /home/ce-dev/deploy/live.local + # This actually does not take any backup, but is needed to populate settings.php. + - mysql_backup: + handling: none + credentials_handling: static + # A list of Drupal sites (for multisites). + - drupal: + sites: + - folder: "default" + public_files: "sites/default/files" + install_command: "-y si localgov" + # Toggle config import on/off. Disabled for initial passes. + config_import_command: "" + # config_import_command: "cim" + config_sync_directory: "config/sync" + sanitize_command: "sql-sanitize" + # Remove after initial pass, to avoid reinstalling Drupal. + force_install: yes + base_url: "https://{{ _domain_name }}" + # Composer command to run. + - composer: + command: install + no_dev: no + working_dir: "{{ deploy_path }}" + apcu_autoloader: no + pre_tasks: + # You can safely remove these steps once you have a working composer.json. + - name: Download composer file. + get_url: + url: https://raw.githubusercontent.com/drupal/recommended-project/9.3.x/composer.json + dest: "{{ deploy_path }}/composer.json" + force: false + - name: Install drush. + command: + cmd: composer require drush/drush:11.* + chdir: "{{ deploy_path }}" + - name: Install localgov. + command: + cmd: composer require localgovdrupal/localgov + chdir: "{{ deploy_path }}" + roles: + - _init # Sets some variables the deploy scripts rely on. + - composer # Composer install step. + - database_backup # This is still needed to generate credentials. + - config_generate # Generates settings.php + # - sync/database_sync # Grab database from a remote server. + - database_apply # Run drush updb and config import. + - _exit # Some common housekeeping. \ No newline at end of file From 223a3b5ecec42997d9e52de690cf7b8c5f9f19ab Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 31 Mar 2022 12:43:36 +0200 Subject: [PATCH 03/40] Adding a new SimpleSAMLphp meta role. (#100) --- roles/_meta/deploy-simplesamlphp/README.md | 9 +++++++++ roles/_meta/deploy-simplesamlphp/tasks/main.yml | 17 +++++++++++++++++ .../deploy_code-simplesamlphp/tasks/main.yml | 1 + 3 files changed, 27 insertions(+) create mode 100644 roles/_meta/deploy-simplesamlphp/README.md create mode 100644 roles/_meta/deploy-simplesamlphp/tasks/main.yml create mode 100644 roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml diff --git a/roles/_meta/deploy-simplesamlphp/README.md b/roles/_meta/deploy-simplesamlphp/README.md new file mode 100644 index 00000000..3d82e671 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/README.md @@ -0,0 +1,9 @@ +# SimpleSAMLphp +Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer. + +This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision: + +* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx + + + diff --git a/roles/_meta/deploy-simplesamlphp/tasks/main.yml b/roles/_meta/deploy-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..7af28fa2 --- /dev/null +++ b/roles/_meta/deploy-simplesamlphp/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Default SimpleSAMLphp role. This is suitable for a standalone SimpleSAMLphp installation + +- ansible.builtin.import_role: + name: _init +- ansible.builtin.import_role: + name: deploy_code +- ansible.builtin.import_role: + name: composer +- ansible.builtin.import_role: + name: database_backup +- ansible.builtin.import_role: + name: live_symlink +- ansible.builtin.import_role: + name: cache_clear/cache_clear-opcache +- ansible.builtin.import_role: + name: _exit diff --git a/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml new file mode 100644 index 00000000..03c03856 --- /dev/null +++ b/roles/deploy_code/deploy_code-simplesamlphp/tasks/main.yml @@ -0,0 +1 @@ +# Nothing to do. \ No newline at end of file From 32843d53a625b68bc3a3f7a8cfcced5c90df3f49 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 13 Apr 2022 13:09:01 +0200 Subject: [PATCH 04/40] Allowing users to set cachetool version properly. (#102) --- roles/cli/cachetool/defaults/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index ad2c6711..5ec1b2d7 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: 8.2.2 \ No newline at end of file + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version \ No newline at end of file diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 5a1969c6..aa27df04 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -13,7 +13,7 @@ when: - deploy_operation == 'deploy' -- name: Download cachetool installer. +- name: Download latest cachetool installer. get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" dest: "{{ cachetool_bin }}" @@ -21,3 +21,14 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists + - cachetool.version == 'latest' + +- name: "Download cachetool version {{ cachetool.version }} installer." + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - deploy_operation == 'deploy' + - not cachetool_global.stat.exists + - cachetool.version != 'latest' From 91ea64efc91984ce88138857898b927b5ffd7b25 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 14 Apr 2022 17:13:09 +0200 Subject: [PATCH 05/40] Deploy ami pr 1.x (#106) * Including an ami.yml for packing AMIs on build finale. * New api_call role, focused on GitLab for now. * Optionally trigger an infra build with an API call. --- roles/api_call/README.md | 9 ++++++++ roles/api_call/defaults/main.yml | 13 +++++++++++ roles/api_call/tasks/main.yml | 35 +++++++++++++++++++++++++++++ roles/deploy_code/defaults/main.yml | 21 +++++++++++++++++ roles/deploy_code/tasks/cleanup.yml | 27 +++++++++++++++++----- roles/deploy_code/tasks/deploy.yml | 10 ++++----- roles/deploy_code/tasks/main.yml | 2 +- 7 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 roles/api_call/README.md create mode 100644 roles/api_call/defaults/main.yml create mode 100644 roles/api_call/tasks/main.yml diff --git a/roles/api_call/README.md b/roles/api_call/README.md new file mode 100644 index 00000000..83d1ab28 --- /dev/null +++ b/roles/api_call/README.md @@ -0,0 +1,9 @@ +# API call + +Making RESTful API calls to other platforms. + + + + + + diff --git a/roles/api_call/defaults/main.yml b/roles/api_call/defaults/main.yml new file mode 100644 index 00000000..091342e8 --- /dev/null +++ b/roles/api_call/defaults/main.yml @@ -0,0 +1,13 @@ +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 \ No newline at end of file diff --git a/roles/api_call/tasks/main.yml b/roles/api_call/tasks/main.yml new file mode 100644 index 00000000..c837df2d --- /dev/null +++ b/roles/api_call/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: Ensure variables are empty. + ansible.builtin.set_fact: + _api_call_variables: "" + _api_call_url: "" + +- name: Build HTML escaped variable string. + ansible.builtin.set_fact: + _api_call_variables: "{{ _api_call_variables + ('' if ansible_loop.first else '&') + 'variables' + item }}" + with_items: "{{ api_call.variables }}" + loop_control: + extended: true + when: api_call.variables | length > 0 + +- name: Build anonymous API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{{ _api_call_variables }}" + when: api_call.token | length == 0 + +- name: Build token authenticated API call URL. + ansible.builtin.set_fact: + _api_call_url: "{{ api_call.base_url }}{{ api_call.path }}?{% if api_call.token_type == 'trigger' %}token={% else %}private_token={% endif %}{{ api_call.token }}&{{ _api_call_variables }}" + when: api_call.token | length > 0 + +- name: Display URL to call. + ansible.builtin.debug: + msg: "{{ _api_call_url }}" + +- name: Make API call. + ansible.builtin.uri: + url: "{{ _api_call_url }}" + method: "{{ api_call.method }}" + return_content: true + status_code: "{{ api_call.status_codes }}" + register: _api_call_return diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index 19e45eee..d616bf08 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -21,3 +21,24 @@ deploy_code: # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index dc72c752..4e648d25 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -1,13 +1,13 @@ --- - name: Ensure codebase is writable. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }} ]; then chmod -R 777 {{ deploy_path_prefix }}{{ item }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} become: true when: "www_user != deploy_user" - name: Ensure permissions are set on directory. - shell: + ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: @@ -15,13 +15,13 @@ - deploy_code.perms_fix_path | length > 1 - name: Delete codebases. - file: + ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} - name: Create a tarball of the deployed codebases. - command: + ansible.builtin.command: cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined @@ -29,7 +29,7 @@ run_once: true - name: Create destination folder. - file: + ansible.builtin.file: path: "{{ deploy_code.mount_sync }}" state: directory mode: "0755" @@ -39,9 +39,24 @@ run_once: true - name: Move to final destination. - command: + ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 run_once: true + +- name: Trigger an infrastructure rebuild. + ansible.builtin.include_role: + name: api_call + vars: + api_call: + type: "{{ deploy_code.api_call.type }}" + base_url: "{{ deploy_code.api_call.base_url }}" + path: "{{ deploy_code.api_call.path }}" + method: "{{ deploy_code.api_call.method }}" + token: "{{ deploy_code.api_call.token }}" + token_type: "{{ deploy_code.api_call.token_type }}" + variables: "{{ deploy_code.api_call.variables }}" + status_codes: "{{ deploy_code.api_call.status_codes }}" + when: deploy_code.rebuild_infra diff --git a/roles/deploy_code/tasks/deploy.yml b/roles/deploy_code/tasks/deploy.yml index 3693eb70..0d15425c 100644 --- a/roles/deploy_code/tasks/deploy.yml +++ b/roles/deploy_code/tasks/deploy.yml @@ -1,6 +1,6 @@ --- - name: Copy project repository. - synchronize: + ansible.posix.synchronize: src: "{{ _ce_deploy_build_dir }}/" dest: "{{ deploy_path }}" archive: true @@ -8,17 +8,17 @@ - "--exclude=.git" - name: Ensure project repository is readable. - file: + ansible.builtin.file: path: "{{ deploy_path }}" state: directory mode: 0755 - name: Project specific tasks. - include_role: + ansible.builtin.include_role: name: "deploy_code/deploy_code-{{ project_type }}" - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ deploy_code.templates }}" @@ -29,7 +29,7 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link diff --git a/roles/deploy_code/tasks/main.yml b/roles/deploy_code/tasks/main.yml index a8021dcf..f4bd05af 100644 --- a/roles/deploy_code/tasks/main.yml +++ b/roles/deploy_code/tasks/main.yml @@ -1,2 +1,2 @@ --- -- include_tasks: "{{ deploy_operation }}.yml" +- ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" From 371b6b9d8b28a908facfe2d80779efd76cb64b40 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 16:20:36 +0200 Subject: [PATCH 06/40] Making the MySQL dump command for routine back-ups less aggressive. (#104) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. --- roles/_init/defaults/main.yml | 1 + .../cron_database_backup-mysql/defaults/main.yml | 1 + .../templates/regular-backups.sh.j2 | 5 +++-- .../database_backup/database_backup-mysql/defaults/main.yml | 1 + .../database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- .../sync/database_sync/database_sync-mysql/defaults/main.yml | 1 + roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 10 insertions(+), 5 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 06470a09..9821f131 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,6 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" +_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index c6c57269..29b8003b 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,3 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index 901cfcf7..fb60cdea 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,8 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - mysqldump -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" + {{ cron_mysql_backup.mysqldump_command }} \ + -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } @@ -23,4 +24,4 @@ cleanup(){ } backup -cleanup \ No newline at end of file +cleanup diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 3ad97b6c..09f44ca3 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,6 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 4b470b89..57a46e2e 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index fe32089a..cbade053 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 48e3e4a0..008f0cf4 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,5 +1,6 @@ --- mysql_sync: + mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index b8182bc3..778092bc 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From a9cc7adf64a8c7959f971df84df9e6172d0cf4c6 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 20 Apr 2022 18:18:47 +0200 Subject: [PATCH 07/40] Fix database backups pr 1.x (#109) * Making the MySQL dump command for routine back-ups less aggressive. * Making max_allowed_packets a variable we can set. * Adding dump command to a re-usable central var. * Renaming _init var so it's easier to distinguish in the code. * Defaults file must be the first param for mysqldump. --- roles/_init/defaults/main.yml | 2 +- .../cron_database_backup-mysql/defaults/main.yml | 2 +- .../cron_database_backup-mysql/templates/regular-backups.sh.j2 | 2 +- roles/database_backup/database_backup-mysql/defaults/main.yml | 2 +- .../database_backup/database_backup-mysql/tasks/deploy-dump.yml | 2 +- .../database_backup-mysql/tasks/deploy-rolling.yml | 2 +- roles/sync/database_sync/database_sync-mysql/defaults/main.yml | 2 +- roles/sync/database_sync/database_sync-mysql/tasks/sync.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 9821f131..c4d15b78 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -2,7 +2,7 @@ # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. deploy_user: "deploy" -_mysqldump_command: "mysqldump --max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml index 29b8003b..f0c869f3 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/defaults/main.yml @@ -2,4 +2,4 @@ cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 index fb60cdea..33c27a5a 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/templates/regular-backups.sh.j2 @@ -10,7 +10,7 @@ TARBALL="$DB_NAME-$(date -Iseconds).sql.bz2" KEEP=$(({{ database.original.backup.keep | default(cron_mysql_backup.keep) }}+1)) backup(){ - {{ cron_mysql_backup.mysqldump_command }} \ + mysqldump {{ cron_mysql_backup.mysqldump_params }} \ -u"$DBUSER" -p"$DBPASSWORD" -h"$DBHOST" "$CURRENT_DBNAME" | bzip2 > "$TARGET_DIR/$TARBALL" ln -sfn "$TARGET_DIR/$TARBALL" "$TARGET_DIR/$DB_NAME" } diff --git a/roles/database_backup/database_backup-mysql/defaults/main.yml b/roles/database_backup/database_backup-mysql/defaults/main.yml index 09f44ca3..f7e070e9 100644 --- a/roles/database_backup/database_backup-mysql/defaults/main.yml +++ b/roles/database_backup/database_backup-mysql/defaults/main.yml @@ -2,7 +2,7 @@ mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index 57a46e2e..d06b1543 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -15,6 +15,6 @@ run_once: true - name: Take a database dump. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index cbade053..1db841b0 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -12,6 +12,6 @@ run_once: true - name: Populate new database. - shell: "{{ mysql_backup.mysqldump_command }} --defaults-extra-file={{ database.credentials_file }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml index 008f0cf4..0ab99e98 100644 --- a/roles/sync/database_sync/database_sync-mysql/defaults/main.yml +++ b/roles/sync/database_sync/database_sync-mysql/defaults/main.yml @@ -1,6 +1,6 @@ --- mysql_sync: - mysqldump_command: "{{ _mysqldump_command }}" # set in _init but you can override here + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index 778092bc..c3ff0b9e 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -67,7 +67,7 @@ when: not database.source.type == 'rolling' - name: Take a dump from source database. - shell: "{{ mysql_sync.mysqldump_command }} --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" + shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - not database.source.type == 'dump' From 4c4c3ad1f9ecd25e25e9c2e5f79d4cb8108c9f34 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 22 Apr 2022 10:28:11 +0100 Subject: [PATCH 08/40] Fix MySQL backup deferral. (#110) --- .../cron_database_backup-mysql/tasks/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml index 40b0c47e..d09fbbac 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/tasks/setup.yml @@ -47,4 +47,4 @@ minute: "{{ _cron_mysql_backup_minute }}" hour: "{{ _cron_mysql_backup_hour }}" job: "{{ _backup_cron_job_command }}" - delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" + delegate_to: "{{ 'localhost' if drupal.defer is defined and drupal.defer else inventory_hostname }}" From 029848d1163191087bd85f372973594b283aad17 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 26 Apr 2022 11:30:31 +0100 Subject: [PATCH 09/40] Files recurse fix pr 1.x (#112) * Don't recurse through site directory when setting permissions during config_generate step. * Update drupal7 config_generate perms update task and use true/false instead of yes/no. --- .../config_generate/config_generate-drupal7/tasks/settings.yml | 3 +-- .../config_generate/config_generate-drupal8/tasks/settings.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index f82c9aa7..277899a0 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -15,9 +15,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index a9937d8e..70d9b113 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -19,9 +19,8 @@ file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory - recurse: true mode: 0775 - become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. template: From b71fa83d6b4deda1daa4532c3a00e3835118cf4a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 29 Apr 2022 11:52:28 +0100 Subject: [PATCH 10/40] Improve multisite support (#115) * Pass -l option in drush commands to specify site name, which should match the site folder name. * Move -l option in drush commands to the first option instead of the last. --- roles/cache_clear/cache_clear-drupal7/tasks/main.yml | 2 +- roles/cache_clear/cache_clear-drupal8/tasks/main.yml | 2 +- .../database_apply-drupal7/tasks/features.yml | 4 ++-- roles/database_apply/database_apply-drupal7/tasks/main.yml | 4 ++-- roles/database_apply/database_apply-drupal8/tasks/main.yml | 6 +++--- .../maintenance_mode-drupal-core/tasks/offline.yml | 4 ++-- .../maintenance_mode-drupal-core/tasks/online.yml | 4 ++-- .../admin_creds/admin_creds-drupal7/tasks/admin.yml | 4 ++-- .../admin_creds/admin_creds-drupal8/tasks/admin.yml | 4 ++-- roles/sanitize/sanitize-drupal8/tasks/main.yml | 2 +- 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml index 1261ff4b..6f72b3a5 100644 --- a/roles/cache_clear/cache_clear-drupal7/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal 7 cache. command: - cmd: "{{ drush_bin }} -y cc all" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml index cfd7fdd3..5b780086 100644 --- a/roles/cache_clear/cache_clear-drupal8/tasks/main.yml +++ b/roles/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Clear Drupal cache. command: - cmd: "{{ drush_bin }} -y cr" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/database_apply/database_apply-drupal7/tasks/features.yml b/roles/database_apply/database_apply-drupal7/tasks/features.yml index 19d58e9b..edba46bf 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/features.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/features.yml @@ -1,8 +1,8 @@ --- - name: Check if Features module is enabled. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} pm-info features | grep ': enabled' | wc -l" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} pm-info features | grep ': enabled' | wc -l" register: features_enabled - name: Revert Drupal configuration from Features. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -y {{ site.revert_features_command }}" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} -y {{ site.revert_features_command }}" when: features_enabled.stdout == "1" diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 048a1457..7c2201e2 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -21,7 +21,7 @@ - name: Apply Drupal database updates. shell: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" with_items: "{{ drupal.sites }}" loop_control: diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 5634faca..dc138585 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -15,7 +15,7 @@ - name: Install Drupal. command: - cmd: "{{ drush_bin }} {{ site.install_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -34,7 +34,7 @@ - name: Apply Drupal database updates. command: - cmd: "{{ drush_bin }} -y updb" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -44,7 +44,7 @@ - name: Import configuration. command: - cmd: "{{ drush_bin }} -y {{ site.config_import_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 2872226a..cbe65b07 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -2,7 +2,7 @@ # @todo this needs refactoring for multisite. - name: Enable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 1 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -12,7 +12,7 @@ - name: Enable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 1" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 99057084..e6a9cbf5 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,7 +1,7 @@ --- - name: Disable maintenance mode. command: - cmd: "{{ drush_bin }} state:set system.maintenance_mode 0 --input-format=integer" + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -10,7 +10,7 @@ - name: Disable maintenance mode D7. shell: - cmd: "{{ drush_bin }} vset maintenance_mode 0" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml index 98401c5c..c8d75bd7 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal7/tasks/admin.yml @@ -6,6 +6,6 @@ _admin_pwd: "{{ lookup('password', '/tmp/{{ project_name }}-{{ site.folder }}-{{ build_type }}-{{ build_number }}-pwd chars=ascii_letters') }}" - name: Reset admin username. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} sql-query \"UPDATE users SET name='{{ _admin_user }}' WHERE uid=1;\"" - name: Reset admin password. - shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file + shell: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} -l {{ site.folder }} upwd {{ _admin_user }} --password='{{ _admin_pwd }}'" \ No newline at end of file diff --git a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml index 0302308b..c2adafdf 100644 --- a/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml +++ b/roles/sanitize/admin_creds/admin_creds-drupal8/tasks/admin.yml @@ -4,14 +4,14 @@ # Loading the user directly is akward, but at least means we don't bypass entity update. - name: Reset admin username. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setUsername(\"{{ admin_creds.username }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.username | length > 1 - name: Reset admin password. command: - cmd: "{{ drush_bin }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " + cmd: "{{ drush_bin }} -l {{ site.folder }} php-eval '$admin = \\Drupal\\user\\Entity\\User::load(1); $admin->setPassword(\"{{ admin_creds.password }}\");$admin->save();' " chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" when: - admin_creds.password | length > 1 diff --git a/roles/sanitize/sanitize-drupal8/tasks/main.yml b/roles/sanitize/sanitize-drupal8/tasks/main.yml index 0a33f5a8..26675f27 100644 --- a/roles/sanitize/sanitize-drupal8/tasks/main.yml +++ b/roles/sanitize/sanitize-drupal8/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Sanitize database. command: - cmd: "{{ drush_bin }} -y {{ site.sanitize_command }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.sanitize_command }}" chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" From c933dc8224e42106df909e29cdb8928f5c5da580 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 23 May 2022 16:19:31 +0100 Subject: [PATCH 11/40] Static credentials handling fix pr 1.x (#119) * Attempt to fix static credentials handling when deploying to an ASG. * Include build_type in static password file location. * Move to using new location for static credentials handling, but try to catch any legacy stuff too, for now. * Add a couple of comments to legacy handling of static credentials. --- .../database_backup-mysql/tasks/deploy.yml | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index a67d240e..dbb39de7 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -24,8 +24,55 @@ - set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' + +### Start of legacy handling. Can remove in a few months. +- name: Catch legacy static password handling. + ansible.builtin.stat: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + register: _legacy_static_creds + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Create new static password location if it doesn't exist. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0755 + delegate_to: localhost + run_once: true + when: mysql_backup.credentials_handling == 'static' + +- name: Copy legacy static password to new location. + ansible.builtin.copy: + src: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + dest: "{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}" + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: 0644 + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists + +- name: Delete legacy static password file if it exists. + ansible.builtin.file: + path: "{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}" + state: absent + delegate_to: localhost + run_once: true + when: + - mysql_backup.credentials_handling == 'static' + - _legacy_static_creds is defined + - _legacy_static_creds.stat.exists +### End of legacy handling. + - set_fact: - _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ inventory_hostname }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" + _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. - set_fact: From b6ea0206e8f83f7e2459babd0622403b955276af Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Tue, 31 May 2022 16:26:10 +0200 Subject: [PATCH 12/40] Making contents of deploy tar 'ownerless'. (#117) --- roles/deploy_code/tasks/cleanup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 4e648d25..03530f22 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -22,7 +22,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 1c2b206d009b21d10b96c434168c15ac5700b96c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 7 Jun 2022 15:48:04 +0100 Subject: [PATCH 13/40] Implement file syncing (#124) * Add a files_sync role. * Clear up some comments, add temp_dir variable and use rsync with command module instead of synchronize module, in files_sync role. * Need trailing slash on src when syncing files to destination server so the contents get synced and not the parent directory itself. * Use rsync instead of copy when syncing files to target server as copy is so slow. * Ensure file sync tasks are run only once. --- roles/sync/files_sync/defaults/main.yml | 16 ++++++++++++++++ roles/sync/files_sync/tasks/main.yml | 7 +++++++ roles/sync/files_sync/tasks/sync.yml | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 roles/sync/files_sync/defaults/main.yml create mode 100644 roles/sync/files_sync/tasks/main.yml create mode 100644 roles/sync/files_sync/tasks/sync.yml diff --git a/roles/sync/files_sync/defaults/main.yml b/roles/sync/files_sync/defaults/main.yml new file mode 100644 index 00000000..5b65b1fb --- /dev/null +++ b/roles/sync/files_sync/defaults/main.yml @@ -0,0 +1,16 @@ +--- +files_sync: + directories: + - source: + # Location of the files to sync from. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_prod/assets/{{ project_name }}_prod_default_public_files" + # Host that contains source files. + host: "localhost" + # Location on deploy server where source files get copied to first. NO TRAILING SLASH. + temp_dir: "/tmp" + # Used to create directory in /tmp. + build_id: mybuildprod + target: + # Location of the files to sync to. DO NOT INCLUDE TRAILING SLASH! + files_dir: "/home/{{ deploy_user }}/shared/{{ project_name }}_dev/assets/{{ project_name }}_dev_default_public_files" + build_id: mybuilddev diff --git a/roles/sync/files_sync/tasks/main.yml b/roles/sync/files_sync/tasks/main.yml new file mode 100644 index 00000000..47c3667a --- /dev/null +++ b/roles/sync/files_sync/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Sync files. + include_tasks: "sync.yml" + with_items: "{{ files_sync.directories }}" + loop_control: + loop_var: files + run_once: true diff --git a/roles/sync/files_sync/tasks/sync.yml b/roles/sync/files_sync/tasks/sync.yml new file mode 100644 index 00000000..e2160008 --- /dev/null +++ b/roles/sync/files_sync/tasks/sync.yml @@ -0,0 +1,21 @@ +--- +- name: Create a temporary directory for source files on localhost. + ansible.builtin.file: + path: "{{ files.source.temp_dir }}/{{ files.source.build_id }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + delegate_to: localhost + run_once: true + +- name: Copy the source files onto the deploy server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.host }}:{{ files.source.files_dir }}/ {{ files.source.temp_dir }}/{{ files.source.build_id }}/" + delegate_to: "localhost" + run_once: true + +- name: Copy the source files from the deploy server onto the destination server. + ansible.builtin.command: + cmd: "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -aHPv {{ files.source.temp_dir }}/{{ files.source.build_id }}/ {{ ansible_play_hosts[0] }}:{{ files.target.files_dir }}/" + delegate_to: "localhost" + run_once: true From 1cd0ed0797c1d26a933b4a3e06e0279e4af6a05a Mon Sep 17 00:00:00 2001 From: EmlynK Date: Wed, 8 Jun 2022 12:47:34 +0100 Subject: [PATCH 14/40] Create Drupal-specific sync roles (#128) * Add a drupal_sync_tasks role to run Drupal specific tasks during syncs. * Move sync database_apply files into tasks subdirectory. * Woops, used bad role names in sync database update roles. * Remove feature reverting from drupal7 database_apply sync role. --- .../cache_clear-drupal7/tasks/main.yml | 11 ++++++++ .../cache_clear-drupal8/tasks/main.yml | 11 ++++++++ .../database_apply-drupal7/tasks/main.yml | 12 +++++++++ .../database_apply-drupal8/tasks/main.yml | 26 +++++++++++++++++++ .../database_apply/tasks/main.yml | 6 +++++ 5 files changed, 66 insertions(+) create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml create mode 100644 roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml new file mode 100644 index 00000000..ab9342ba --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal7/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal 7 cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cc all" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml new file mode 100644 index 00000000..96c721ee --- /dev/null +++ b/roles/sync/drupal_sync_tasks/cache_clear/cache_clear-drupal8/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Clear Drupal cache. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y cr" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + run_once: true diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml new file mode 100644 index 00000000..f520514e --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal7/tasks/main.yml @@ -0,0 +1,12 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.shell: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml new file mode 100644 index 00000000..bb9cc73c --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Apply Drupal database updates. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + +- name: Import configuration. + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" + chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + become: "{{ 'no' if www_user == deploy_user else 'yes' }}" + become_user: "{{ www_user }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site + when: + - site.config_import_command + +- name: Clear the cache. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/cache_clear/cache_clear-{{ project_type }}" diff --git a/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml new file mode 100644 index 00000000..58f6215a --- /dev/null +++ b/roles/sync/drupal_sync_tasks/database_apply/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Apply database updates. + ansible.builtin.include_role: + name: "sync/drupal_sync_tasks/database_apply/database_apply-{{ project_type }}" + when: deploy_operation == 'deploy' + run_once: true From 904033f8bf3e1973d7b78c8ea2072426f1d0decb Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 10 Jun 2022 12:57:40 +0200 Subject: [PATCH 15/40] Fixing GRANT query for MySQL > 8.0. (#131) * Fixing GRANT query for MySQL > 8.0. * Create what though? --- roles/_init/defaults/main.yml | 1 + roles/database_backup/database_backup-mysql/tasks/deploy.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index c4d15b78..790e9409 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -1,6 +1,7 @@ --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" _mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index dbb39de7..b0328958 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -88,8 +88,9 @@ # to allow this role to be looped over, # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. +# As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 94715d8bd5363c640649e09bffe0e3b1574ceb90 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Mon, 13 Jun 2022 10:34:12 +0100 Subject: [PATCH 16/40] Use IF NOT EXISTS when creating database user as that command fails if static database users are being used. (#133) --- roles/database_backup/database_backup-mysql/tasks/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index b0328958..f5a7130a 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -90,7 +90,7 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true From 42f12c81d0974648857d203139a154c736ac9731 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Thu, 16 Jun 2022 13:39:30 +0100 Subject: [PATCH 17/40] Attempt to fix syncs whenever the 'dump' type is used for source or target. (#149) --- .../database_sync-mysql/tasks/sync.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml index c3ff0b9e..98eca0f3 100644 --- a/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml +++ b/roles/sync/database_sync/database_sync-mysql/tasks/sync.yml @@ -38,12 +38,6 @@ - name: Register remote dump name (from database). set_fact: mysql_sync_source_dump_path: "/tmp/{{ database.source.database }}.sql.bz2" - when: not database.source.type == 'dump' - -- name: Register existing remote dump name. - set_fact: - mysql_sync_source_dump_path: "{{ database.source.database }}" - when: database.source.type == 'dump' - name: Get source last known good build number. command: @@ -70,7 +64,6 @@ shell: "mysqldump --defaults-extra-file={{ database.source.credentials_file }} {{ mysql_sync.mysqldump_params }} {{ mysql_sync_source_database }} | bzip2 > {{ mysql_sync_source_dump_path }}" delegate_to: "{{ database.source.host }}" when: - - not database.source.type == 'dump' - database.source.fresh_db - name: Find source database host. @@ -97,12 +90,6 @@ - name: Register tmp target dump name. set_fact: mysql_sync_target_dump_path: "/tmp/{{ database.target.database }}.sql.bz2" - when: not database.target.type == 'dump' - -- name: Register final target dump name. - set_fact: - mysql_sync_target_dump_path: "{{ database.target.database }}" - when: database.target.type == 'dump' - name: Get target last known good build number. command: @@ -139,21 +126,17 @@ - name: Drop target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'drop database if exists {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Recreate target database. shell: "mysql --defaults-extra-file={{ database.target.credentials_file }} -e 'create database {{ mysql_sync_target_database }};'" - when: not database.target.type == 'dump' - name: Repopulate database from dump. shell: "bzcat {{ mysql_sync_target_dump_path }} | mysql --defaults-extra-file={{ database.target.credentials_file }} {{ mysql_sync_target_database }}" - when: not database.target.type == 'dump' - name: Remove tmp dump file. file: path: "{{ mysql_sync_target_dump_path }}" state: absent - when: not database.target.type == 'dump' - name: Enable all autoscale processes on source ASG. ansible.builtin.command: > From a9c6abfb5c686d030a0671d8a20805b7d3b1be02 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 20 Jun 2022 11:31:01 +0200 Subject: [PATCH 18/40] Squashfs pr 1.x (#150) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. --- roles/_init/tasks/main.yml | 51 +++++++++---- roles/deploy_code/defaults/main.yml | 13 +++- roles/deploy_code/tasks/cleanup.yml | 108 +++++++++++++++++++++++++++- roles/live_symlink/tasks/main.yml | 23 ++++-- 4 files changed, 171 insertions(+), 24 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index a1b6e204..53be27d3 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -1,27 +1,45 @@ --- # Ensure default values for common variables. - name: Define deploy user. - set_fact: + ansible.builtin.set_fact: deploy_user: "{{ deploy_user | default('deploy') }}" - name: Define deploy base path. - set_fact: + ansible.builtin.set_fact: deploy_base_path: "{{ deploy_base_path | default('/home/{{ deploy_user }}/deploy/{{ project_name }}_{{ build_type }}') }}" - name: Define mounted directory for assets. - set_fact: + ansible.builtin.set_fact: deploy_assets_base_path: "{{ deploy_assets_base_path | default('/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/assets') }}" - name: Define webroot. - set_fact: + ansible.builtin.set_fact: webroot: "{{ webroot | default('web') }}" - name: Define build deploy path prefix. - set_fact: + ansible.builtin.set_fact: deploy_path_prefix: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_" - name: Define build deploy path. - set_fact: + ansible.builtin.set_fact: deploy_path: "{{ deploy_path | default('{{ deploy_path_prefix }}{{ build_number }}') }}" - name: Define live_symlink dest. - set_fact: + ansible.builtin.set_fact: live_symlink_dest: "{{ live_symlink_dest | default('{{ deploy_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" +# Manipulate variables for SquashFS builds. +- name: Define image builds base path. + ansible.builtin.set_fact: + build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" + when: deploy_code.mount_type == "squashfs" +- name: Define image builds build path prefix. + ansible.builtin.set_fact: + build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" + when: deploy_code.mount_type == "squashfs" +- name: Define live_symlink dest for image builds. + ansible.builtin.set_fact: + live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" + when: deploy_code.mount_type == "squashfs" +- name: Overwrite deploy and live_symlink paths if SquashFS deploy. + ansible.builtin.set_fact: + deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" + when: deploy_code.mount_type == "squashfs" + # Gather last known good build directly from symlink. # This can happen: # - when the first builds failed, @@ -42,7 +60,7 @@ # This is passed from caller. - name: Gather last known good build number. - set_fact: + ansible.builtin.set_fact: previous_build_number: "{{ previous_known_build_number }}" # - set_fact: @@ -53,27 +71,34 @@ # Make sure the deploy target exists. - name: Ensure deploy target directory exists. - file: + ansible.builtin.file: path: "{{ deploy_base_path }}" state: directory +# Make sure the build target exists. +- name: Ensure build target directory exists. + ansible.builtin.file: + path: "{{ build_base_path }}" + state: directory + when: deploy_code.mount_type == "squashfs" + # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. - stat: + ansible.builtin.stat: path: "{{ role_path }}/tasks/{{ project_type }}.yml" register: _project_type_task_result delegate_to: "localhost" # Project specific init tasks. - name: Include project init tasks. - include_tasks: "{{ project_type }}.yml" + ansible.builtin.include_tasks: "{{ project_type }}.yml" when: - _project_type_task_result.stat.exists - name: Define opcache cachetool path. - set_fact: + ansible.builtin.set_fact: cachetool_bin: "{{ cachetool_bin | default('/home/{{ deploy_user }}/.bin/cachetool.phar') }}" - name: Ensure we have a cachetool binary. - import_role: + ansible.builtin.import_role: name: cli/cachetool diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index d616bf08..f1f7cdb4 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -3,7 +3,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -15,12 +15,21 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false # Details of API call to trigger. See api_call role. diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 03530f22..c722b049 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -6,19 +6,39 @@ become: true when: "www_user != deploy_user" -- name: Ensure permissions are set on directory. +- name: Ensure permissions are set on deploy directory. ansible.builtin.shell: cmd: "if [ -d {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ deploy_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} when: - deploy_code.perms_fix_path is defined - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type != "squashfs" -- name: Delete codebases. +- name: Ensure permissions are set on builds directory. + ansible.builtin.shell: + cmd: "if [ -d {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }} ]; then chmod 755 {{ build_path_prefix }}{{ item }}/{{ deploy_code.perms_fix_path }}; fi" + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.perms_fix_path is defined + - deploy_code.perms_fix_path | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Delete codebases from deploy directory. ansible.builtin.file: name: "{{ deploy_path_prefix }}{{ item }}" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type != "squashfs" + +- name: Delete codebases from builds directory. + ansible.builtin.file: + name: "{{ build_path_prefix }}{{ item }}" + state: absent + with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - deploy_code.keep, 0] | max }} + when: + - deploy_code.mount_type == "squashfs" - name: Create a tarball of the deployed codebases. ansible.builtin.command: @@ -26,6 +46,26 @@ when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Ensure older SquashFS images are deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + run_once: true + +- name: Create a SquashFS image of the deployed codebases. + ansible.builtin.command: + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true - name: Create destination folder. @@ -38,14 +78,76 @@ - deploy_code.mount_sync | length > 1 run_once: true -- name: Move to final destination. +- name: Move tar file to final destination. ansible.builtin.command: cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "tarball" + run_once: true + +- name: Move SquashFS image to final destination. + ansible.builtin.command: + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" run_once: true +- name: Copy SquashFS image to local server. + ansible.builtin.command: + cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Check if we have a mount already. + ansible.builtin.shell: + cmd: "mount | grep {{ deploy_base_path }}" + ignore_errors: true + register: _mount_check + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount -l {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + +- name: Reload any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: reloaded + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _mount_check is succeeded + - deploy_code.services | length > 0 + +- name: Mount new SquashFS image. + ansible.builtin.command: + cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: name: api_call diff --git a/roles/live_symlink/tasks/main.yml b/roles/live_symlink/tasks/main.yml index 460ced9a..fa1d9e97 100644 --- a/roles/live_symlink/tasks/main.yml +++ b/roles/live_symlink/tasks/main.yml @@ -1,19 +1,28 @@ --- -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _live_symlink_build_target: "{{ deploy_base_path }}/{{ project_name }}_{{ build_type }}_build_{{ previous_build_number }}" when: deploy_operation == 'revert' +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_dest }}" + +- ansible.builtin.set_fact: + _live_symlink_dest_target: "{{ live_symlink_build_dest }}" + when: deploy_code.mount_type == "squashfs" + - name: Symlink build. - file: + ansible.builtin.file: src: "{{ _live_symlink_build_target }}" - dest: "{{ live_symlink_dest }}" + dest: "{{ _live_symlink_dest_target }}" state: link + follow: false + force: true - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ live_symlink.templates }}" @@ -24,10 +33,12 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link + follow: false + force: true with_items: "{{ live_symlink.symlinks }}" loop_control: loop_var: link From a4de7710d2d7609aa9213c1f73951eaf27b23cbd Mon Sep 17 00:00:00 2001 From: EmlynK Date: Fri, 24 Jun 2022 13:00:24 +0100 Subject: [PATCH 19/40] Check deploy_code.mount_type is defined when setting facts in init role. (#155) --- roles/_init/tasks/main.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/roles/_init/tasks/main.yml b/roles/_init/tasks/main.yml index 53be27d3..effa2790 100644 --- a/roles/_init/tasks/main.yml +++ b/roles/_init/tasks/main.yml @@ -26,19 +26,27 @@ - name: Define image builds base path. ansible.builtin.set_fact: build_base_path: "/home/{{ deploy_user }}/builds/{{ project_name }}_{{ build_type }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define image builds build path prefix. ansible.builtin.set_fact: build_path_prefix: "{{ build_base_path }}/{{ project_name }}_{{ build_type }}_build_" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Define live_symlink dest for image builds. ansible.builtin.set_fact: live_symlink_build_dest: "{{ live_symlink_build_dest | default('{{ build_base_path }}/live.{{ project_name }}_{{ build_type }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" - name: Overwrite deploy and live_symlink paths if SquashFS deploy. ansible.builtin.set_fact: deploy_path: "{{ build_path | default('{{ build_path_prefix }}{{ build_number }}') }}" - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Gather last known good build directly from symlink. # This can happen: @@ -80,7 +88,9 @@ ansible.builtin.file: path: "{{ build_base_path }}" state: directory - when: deploy_code.mount_type == "squashfs" + when: + - deploy_code.mount_type is defined + - deploy_code.mount_type == "squashfs" # Check for project specific init tasks. - name: Check that {{ project_type }}.yml exists. From fe2aaf10f5bfc9124f5c1ab1cb6bff6e8e5b635c Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 28 Jun 2022 11:08:07 +0100 Subject: [PATCH 20/40] Make config imports during syncs optional (#157) * Use a different variable for config imports during a sync so they can be optional. * Actually, no. Use a variable to determine if config should be imported during a sync. * Fix some logic with config import during syncs and add a comment to explain when the task runs. --- .../database_apply/database_apply-drupal8/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml index bb9cc73c..e050af3f 100644 --- a/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/sync/drupal_sync_tasks/database_apply/database_apply-drupal8/tasks/main.yml @@ -9,6 +9,7 @@ loop_control: loop_var: site +# This only runs if the sync_config_import variable is not defined or it is defined and is true. If it's defined and false, this won't run. - name: Import configuration. ansible.builtin.command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y {{ site.config_import_command }}" @@ -20,6 +21,7 @@ loop_var: site when: - site.config_import_command + - site.sync_config_import is not defined or site.sync_config_import - name: Clear the cache. ansible.builtin.include_role: From 451227d28461c0afcc0bb854277fda17796d1d9e Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 29 Jun 2022 17:40:45 +0200 Subject: [PATCH 21/40] Squashfs pr 1.x (#153) * Adding SquashFS option to syncing. * Make 'tarball' the default mount type so nothing existing breaks. * Altering deploy_path so we can build in another location for SquashFS builds. * Ensuring the build dir exists if doing a SquashFS build. * Force symlink creation as the deploy dir may not exist yet. * Running mount commands with sudo. * Tweaking link command and destination for live links. * Slight bug in link path handling for SquashFS. * Checking for existing mount and using remount operation. * Stop ce-deploy trying to delete from read-only SquashFS mount. * Formatting error, these special jinja2 things are not filters - no spaces. * Reloading services to make sure mounting doesn't fail. * Picking more sensible loop var name. * Moving to shell for mount check and fixing jinja2 filter names. * Trying with the posix mount module instead of command. * Working through user/sudo issues. * Altering symlink handling slightly so we have the deploy directory always set. * Simplifying 'when' checks. * Adding a revert behaviour for SquashFS. * Making some wording a little less ambiguous. * Fixing bad image path and adding clauses to stat check. * Trying to make remounting SquashFS images a bit safer. * Adding when clauses to SquashFS image path check. * Removing revert code, as it cannot work. --- roles/deploy_code/tasks/cleanup.yml | 64 +++++++++++++++++++---------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index c722b049..e1412749 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -49,16 +49,6 @@ - deploy_code.mount_type == "tarball" run_once: true -- name: Ensure older SquashFS images are deleted. - ansible.builtin.file: - path: "{{ build_base_path }}/deploy.sqsh" - state: absent - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - run_once: true - - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" @@ -96,6 +86,36 @@ - deploy_code.mount_type == "squashfs" run_once: true +- name: Check if we have a SquashFS image already there. + ansible.builtin.stat: + path: "{{ build_base_path }}/deploy.sqsh" + register: _deploy_code_mount_image + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + +- name: Copy previous SquashFS image in case of rollback. + ansible.builtin.copy: + remote_src: true + force: true + src: "{{ build_base_path }}/deploy.sqsh" + dest: "{{ build_base_path }}/deploy_previous.sqsh" + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_image.stat.islnk is defined + +- name: Ensure mounted SquashFS image is deleted. + ansible.builtin.file: + path: "{{ build_base_path }}/deploy.sqsh" + state: absent + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - name: Copy SquashFS image to local server. ansible.builtin.command: cmd: "cp {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh {{ build_base_path }}/deploy.sqsh" @@ -108,21 +128,11 @@ ansible.builtin.shell: cmd: "mount | grep {{ deploy_base_path }}" ignore_errors: true - register: _mount_check - when: - - deploy_code.mount_sync is defined - - deploy_code.mount_sync | length > 1 - - deploy_code.mount_type == "squashfs" - -- name: Unmount existing SquashFS image. - ansible.builtin.command: - cmd: "umount -l {{ deploy_base_path }}" - become: true + register: _deploy_code_mount_check when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded - name: Reload any services that might be keeping the loop device busy. ansible.builtin.service: @@ -136,9 +146,19 @@ - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - - _mount_check is succeeded + - _deploy_code_mount_check is succeeded - deploy_code.services | length > 0 +- name: Unmount existing SquashFS image. + ansible.builtin.command: + cmd: "umount {{ deploy_base_path }}" + become: true + when: + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - name: Mount new SquashFS image. ansible.builtin.command: cmd: "mount {{ build_base_path }}/deploy.sqsh {{ deploy_base_path }} -t squashfs -o loop" From c7aae71b8903846fccbbd325464509282cb6a302 Mon Sep 17 00:00:00 2001 From: EmlynK Date: Tue, 16 Aug 2022 12:18:58 +0100 Subject: [PATCH 22/40] Add cache clears to Drupal deployments, before DB updates and stuff (#159) * Add cache clears to Drupal deployments, before database updates and config import are run. * Move cache clears out of meta roles and only run in database_apply role on non-initial builds. * Add cache clear stuff to drupal7 database_apply role. --- roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++ roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 7c2201e2..8b655bda 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -19,6 +19,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index dc138585..23cb0b3b 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -32,6 +32,11 @@ loop_var: site when: previous_build_number == 0 or (site.force_install is defined and site.force_install) +- name: Clear the cache. + ansible.builtin.include_role: + name: "cache_clear/cache_clear-{{ project_type }}" + when: previous_build_number > 0 + - name: Apply Drupal database updates. command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 15b8daae0b9736f2a80878415a02155dd2acc8e0 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 5 Sep 2022 18:05:10 +0200 Subject: [PATCH 23/40] Avoid leaving exponentially growing sqsh files in build locations! (#164) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. --- roles/deploy_code/tasks/cleanup.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index e1412749..b206b7ab 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -95,17 +95,18 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" -- name: Copy previous SquashFS image in case of rollback. +- name: Copy previous SquashFS image to shared mount point in case of rollback. ansible.builtin.copy: remote_src: true force: true src: "{{ build_base_path }}/deploy.sqsh" - dest: "{{ build_base_path }}/deploy_previous.sqsh" + dest: "{{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}_previous.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" - _deploy_code_mount_image.stat.islnk is defined + run_once: true - name: Ensure mounted SquashFS image is deleted. ansible.builtin.file: From 5cab4ae00d301e183ab1d17cf53bf289391b4a6b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Thu, 6 Oct 2022 16:28:30 +0200 Subject: [PATCH 24/40] Exclude sqsh file pr 1.x (#167) * Avoid leaving exponentially growing sqsh files in build locations! * Use mount point instead of /tmp for stowing previous sqsh file. * Making sure image/tarball filenames are build specific in /tmp. --- roles/deploy_code/tasks/cleanup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index b206b7ab..353854b3 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -42,7 +42,7 @@ - name: Create a tarball of the deployed codebases. ansible.builtin.command: - cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}.tar --owner=0 --group=0 {{ deploy_base_path }}" + cmd: "tar -cvf /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar --owner=0 --group=0 {{ deploy_base_path }}" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -51,7 +51,7 @@ - name: Create a SquashFS image of the deployed codebases. ansible.builtin.command: - cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}.sqsh -e {{ build_base_path }}/deploy.sqsh" + cmd: "mksquashfs {{ build_base_path }} /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh -e {{ build_base_path }}/deploy.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -70,7 +70,7 @@ - name: Move tar file to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.tar {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.tar" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 @@ -79,7 +79,7 @@ - name: Move SquashFS image to final destination. ansible.builtin.command: - cmd: "mv /tmp/{{ project_name }}_{{ build_type }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" + cmd: "mv /tmp/{{ project_name }}_{{ build_type }}_{{ build_number }}.sqsh {{ deploy_code.mount_sync }}/{{ project_name }}_{{ build_type }}.sqsh" when: - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 From 459faba5e163ee6355d4bb56f9ad19042c7aa148 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 7 Oct 2022 16:05:35 +0200 Subject: [PATCH 25/40] Removing unnecessary lines in Drupal config generation. (#169) * Removing unnecessary lines in Drupal config generation. * Drupal 7 lacked the install_command var. --- .../tasks/settings.yml | 16 ++++++--------- .../tasks/settings.yml | 20 ++++++++----------- .../database_apply-drupal7/tasks/main.yml | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/roles/config_generate/config_generate-drupal7/tasks/settings.yml b/roles/config_generate/config_generate-drupal7/tasks/settings.yml index 277899a0..f8e3b55e 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/settings.yml @@ -1,25 +1,21 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set private files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/config_generate/config_generate-drupal8/tasks/settings.yml b/roles/config_generate/config_generate-drupal8/tasks/settings.yml index 70d9b113..ebf182e8 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/settings.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/settings.yml @@ -1,29 +1,25 @@ --- -- name: set public files relative path. - set_fact: +- name: Set public files relative path. + ansible.builtin.set_fact: build_public_file_path: "{{ site.public_files }}" -- name: set public files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" -- name: set private files path. - set_fact: - build_private_file_path: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_{{ site.folder }}_private_files" - -- name: set config directory path. - set_fact: +- name: Set config directory path. + ansible.builtin.set_fact: build_config_sync_directory: "{{ deploy_path }}/{{ site.config_sync_directory }}" - name: Ensure file permissions on site folder. - file: + ansible.builtin.file: path: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" state: directory mode: 0775 become: "{{ false if www_user == deploy_user else true }}" - name: Generates settings.php file. - template: + ansible.builtin.template: src: "{{ item }}" dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/settings.php" with_first_found: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index 8b655bda..eb2edbfa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install Drupal. shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} -y si" + cmd: "{{ drush_bin }} -l {{ site.folder }} {{ site.install_command }}" chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -45,4 +45,4 @@ include_tasks: ctools.yml with_items: "{{ drupal.sites }}" loop_control: - loop_var: site \ No newline at end of file + loop_var: site From 200f59002c0459d63edc4d821f7b31ee94dde351 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 10 Oct 2022 14:31:48 +0200 Subject: [PATCH 26/40] Ensuring dump directory exists on backup step. (#172) * Ensuring dump directory exists on backup step. * Acts on shared storage, so can run once. --- .../database_backup-mysql/tasks/deploy-dump.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml index d06b1543..80a6e5ef 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-dump.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" @@ -14,7 +14,15 @@ when: previous_build_number == 0 run_once: true +- name: Ensure the dump directory exists. + ansible.builtin.file: + path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + run_once: true + - name: Take a database dump. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ database.database }} | bzip2 > {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2" when: previous_build_number > 0 run_once: true From 2c7443277b94486bad5f3d7b57212c9426bf5e9f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 13:49:41 +0200 Subject: [PATCH 27/40] Allowing Drupal 7 jobs to disable cron. (#174) --- roles/cron/cron_drupal7/defaults/main.yml | 1 + roles/cron/cron_drupal7/tasks/cron.yml | 2 +- roles/cron/cron_drupal7/tasks/job.yml | 10 ++++++---- roles/cron/cron_drupal7/tasks/main.yml | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index a09c1036..a6551ab6 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -8,6 +8,7 @@ drupal: - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] diff --git a/roles/cron/cron_drupal7/tasks/cron.yml b/roles/cron/cron_drupal7/tasks/cron.yml index 2bc2c6c7..056d50e0 100644 --- a/roles/cron/cron_drupal7/tasks/cron.yml +++ b/roles/cron/cron_drupal7/tasks/cron.yml @@ -1,6 +1,6 @@ --- - name: Setup Drupal 7 cron tasks. - include_tasks: + ansible.builtin.include_tasks: file: job.yml with_items: "{{ site.cron }}" loop_control: diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index 841c4b51..0f1d3203 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -1,10 +1,10 @@ --- - name: Define cron job command. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\"" when: - drupal.defer is defined @@ -13,7 +13,7 @@ - drupal.defer_target | length > 0 - name: Define cron job command for differing deploy users. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'" when: - www_user != deploy_user @@ -21,9 +21,11 @@ - drupal.defer - name: Setup Drupal cron tasks on app server. - cron: + ansible.builtin.cron: name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" job: "{{ _cron_job_command }}" + state: present + disabled: "{{ entry.disabled | default(omit) }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal7/tasks/main.yml b/roles/cron/cron_drupal7/tasks/main.yml index 75fa05c7..cf0f80f6 100644 --- a/roles/cron/cron_drupal7/tasks/main.yml +++ b/roles/cron/cron_drupal7/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -10,7 +10,7 @@ - drupal.defer - name: Create site cron entries. - include_tasks: + ansible.builtin.include_tasks: file: cron.yml with_items: "{{ drupal.sites }}" loop_control: From 03852117a42d9aa62e30d3161ae0b418d6e440bf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 14 Oct 2022 17:41:52 +0200 Subject: [PATCH 28/40] Suppress db revert pr 1.x (#177) * Supporting option to suppress reverting backups. * Fixing namespacing in database_backup role. --- .../tasks/cleanup-dump.yml | 2 +- .../tasks/cleanup-rolling.yml | 2 +- .../database_backup-mysql/tasks/cleanup.yml | 12 ++++---- .../tasks/deploy-none.yml | 6 ++-- .../tasks/deploy-rolling.yml | 8 +++--- .../database_backup-mysql/tasks/deploy.yml | 28 +++++++++---------- .../database_backup-mysql/tasks/main.yml | 2 +- .../tasks/revert-dump.yml | 2 +- .../database_backup-mysql/tasks/revert.yml | 14 ++++++---- roles/database_backup/defaults/main.yml | 1 + roles/database_backup/tasks/main.yml | 4 +-- 11 files changed, 42 insertions(+), 39 deletions(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml index d5774a78..a9ea9ae2 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-dump.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql dumps. - file: + ansible.builtin.file: path: "{{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ item }}.sql.bz2" state: absent with_sequence: start={{ [previous_build_number | int - 50, 0] | max }} end={{ [previous_build_number | int - mysql_backup.keep, 0] | max }} diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml index b131f8e6..2bdf29ee 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup-rolling.yml @@ -1,7 +1,7 @@ --- # We assume it's safe to only go back 50 build back. - name: Delete mysql databases. - mysql_db: + community.mysql.mysql_db: name: "{{ database.database }}_{{ item }}" state: absent login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml index 71570baa..ac71fcbb 100644 --- a/roles/database_backup/database_backup-mysql/tasks/cleanup.yml +++ b/roles/database_backup/database_backup-mysql/tasks/cleanup.yml @@ -1,23 +1,23 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "cleanup-{{ mysql_backup.handling }}.yml" # We assume it's safe to only go back 50 build back. - name: Delete mysql users. - mysql_user: + community.mysql.mysql_user: name: "{{ database.user }}_{{ item }}" host_all: true state: absent diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml index 65c68978..dea8fe9f 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-none.yml @@ -1,11 +1,11 @@ --- -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}" - name: Create initial database. - mysql_db: + community.mysql.mysql_db: name: "{{ _mysql_build_database_name }}" state: present login_host: "{{ _mysql_host }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml index 1db841b0..71fa7580 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy-rolling.yml @@ -1,17 +1,17 @@ --- # Setup database names. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_database_name: "{{ database.database }}_{{ build_number }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_previous_build_database_name: "{{ database.database }}_{{ previous_build_number }}" # Note: we don't use the mysql_db Ansible module on purpose. # If database already exists, we want to fail and not override it # with previous build. - name: Create new database. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE DATABASE {{ _mysql_build_database_name }};" run_once: true - name: Populate new database. - shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" + ansible.builtin.shell: "mysqldump --defaults-extra-file={{ database.credentials_file }} {{ mysql_backup.mysqldump_params }} {{ _mysql_previous_build_database_name }} | mysql --defaults-extra-file={{ database.credentials_file }} {{ _mysql_build_database_name }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index f5a7130a..fc06be74 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -1,27 +1,27 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" # Credentials for the app itself. # Manual: reuse given creds. Note the user gets ignored. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ _mysql_user }}" when: mysql_backup.credentials_handling == 'manual' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ _mysql_password }}" when: mysql_backup.credentials_handling == 'manual' # Static: create/update user/pwd only once. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}" when: mysql_backup.credentials_handling == 'static' @@ -71,18 +71,18 @@ - _legacy_static_creds.stat.exists ### End of legacy handling. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '{{ _ce_deploy_data_dir }}/{{ project_name }}_{{ build_type }}/mysql/{{ _mysql_host }}/{{ database.database }}') }}" when: mysql_backup.credentials_handling == 'static' # Rotate: create user/pwd on each build. -- set_fact: +- ansible.builtin.set_fact: _mysql_build_user_name: "{{ database.user }}_{{ build_number }}" when: mysql_backup.credentials_handling == 'rotate' -- set_fact: +- ansible.builtin.set_fact: _mysql_build_password: "{{ lookup('password', '/dev/shm/{{ project_name }}_{{ build_type }}_{{ build_number }}') }}" when: mysql_backup.credentials_handling == 'rotate' -- include_tasks: "deploy-{{ mysql_backup.handling }}.yml" +- ansible.builtin.include_tasks: "deploy-{{ mysql_backup.handling }}.yml" # We append privileges instead of replacing, # to allow this role to be looped over, @@ -90,11 +90,11 @@ # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. - name: Create/update mysql user. - command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true -- set_fact: +- ansible.builtin.set_fact: _build_database: original: "{{ database }}" name: "{{ _mysql_build_database_name }}" @@ -102,5 +102,5 @@ password: "{{ _mysql_build_password }}" host: "{{ _mysql_host }}" -- set_fact: +- ansible.builtin.set_fact: build_databases: "{{ build_databases + [ _build_database ] }}" diff --git a/roles/database_backup/database_backup-mysql/tasks/main.yml b/roles/database_backup/database_backup-mysql/tasks/main.yml index ec10df75..b015ac09 100644 --- a/roles/database_backup/database_backup-mysql/tasks/main.yml +++ b/roles/database_backup/database_backup-mysql/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Take database backup. - include_tasks: "{{ deploy_operation }}.yml" + ansible.builtin.include_tasks: "{{ deploy_operation }}.yml" with_items: "{{ mysql_backup.databases }}" loop_control: loop_var: database diff --git a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml index 735a9805..f914ee78 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert-dump.yml @@ -1,5 +1,5 @@ --- - name: Revert database from dump. - shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" + ansible.builtin.shell: "bzcat {{ mysql_backup.dumps_directory }}/{{ _mysql_host }}/{{ database.database }}-{{ previous_build_number }}.sql.bz2 | mysql --defaults-extra-file={{ database.credentials_file }} {{ database.database }}" when: previous_build_number > 0 run_once: true diff --git a/roles/database_backup/database_backup-mysql/tasks/revert.yml b/roles/database_backup/database_backup-mysql/tasks/revert.yml index ba054b44..f6076262 100644 --- a/roles/database_backup/database_backup-mysql/tasks/revert.yml +++ b/roles/database_backup/database_backup-mysql/tasks/revert.yml @@ -1,17 +1,19 @@ --- - name: Grab mysql credentials. - fetch: + ansible.builtin.fetch: src: "{{ database.credentials_file }}" dest: "{{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini" flat: true # Credentials for "managing" databases. -- set_fact: +- ansible.builtin.set_fact: _mysql_host: "{{ lookup('ini', 'host section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_user: "{{ lookup('ini', 'user section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- set_fact: +- ansible.builtin.set_fact: _mysql_password: "{{ lookup('ini', 'password section=client file={{ _ce_deploy_build_dir }}/mysql_backup_credentials.ini') }}" -- include_tasks: "revert-{{ mysql_backup.handling }}.yml" - when: previous_build_number > 0 +- ansible.builtin.include_tasks: "revert-{{ mysql_backup.handling }}.yml" + when: + - previous_build_number > 0 + - database_backup.revert diff --git a/roles/database_backup/defaults/main.yml b/roles/database_backup/defaults/main.yml index 9201fa20..df4881c0 100644 --- a/roles/database_backup/defaults/main.yml +++ b/roles/database_backup/defaults/main.yml @@ -2,3 +2,4 @@ database_backup: engines: - mysql + revert: true \ No newline at end of file diff --git a/roles/database_backup/tasks/main.yml b/roles/database_backup/tasks/main.yml index 83adedf9..829536c2 100644 --- a/roles/database_backup/tasks/main.yml +++ b/roles/database_backup/tasks/main.yml @@ -1,9 +1,9 @@ --- -- set_fact: +- ansible.builtin.set_fact: build_databases: [] - name: Take database dumps. - include_role: + ansible.builtin.include_role: name: "database_backup/database_backup-{{ engine }}" with_items: "{{ database_backup.engines }}" loop_control: From 86f138a15aafe6dd75b2eba1b04ceafb39351782 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 18 Nov 2022 15:05:04 +0100 Subject: [PATCH 29/40] Fixing bad assumption that databases will have TCP connections. (#179) --- .../database_backup/database_backup-mysql/tasks/deploy.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/database_backup/database_backup-mysql/tasks/deploy.yml b/roles/database_backup/database_backup-mysql/tasks/deploy.yml index fc06be74..5442230d 100644 --- a/roles/database_backup/database_backup-mysql/tasks/deploy.yml +++ b/roles/database_backup/database_backup-mysql/tasks/deploy.yml @@ -89,11 +89,16 @@ # for multisites or projects with multiple databases. # @see https://www.thesysadmin.rocks/2020/10/08/rds-mariadb-grant-all-permission-access-denied-for-user/ for why we cannot GRANT ALL. # As of MySQL 8.0 the GRANT operation has no password option, you must CREATE your user first. -- name: Create/update mysql user. +- name: Create/update mysql user for TCP connections. ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'%' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'%';" when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) run_once: true +- name: Create/update mysql user for unix socket connections. + ansible.builtin.command: mysql --defaults-extra-file={{ database.credentials_file }} -e "CREATE USER IF NOT EXISTS '{{ _mysql_build_user_name }}'@'localhost' IDENTIFIED BY '{{ _mysql_build_password }}'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON {{ _mysql_build_database_name }}.* TO '{{ _mysql_build_user_name }}'@'localhost';" + when: ( mysql_backup.credentials_handling == 'rotate' ) or ( mysql_backup.credentials_handling == 'static' ) + run_once: true + - ansible.builtin.set_fact: _build_database: original: "{{ database }}" From b23654081e8f8a5058d97b2e8b17a785d9a548f2 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 16:05:49 +0100 Subject: [PATCH 30/40] Handling the 'drush deploy' command more elegantly for Drupal 8+. (#180) * Handling the 'drush deploy' command more elegantly for Drupal 8+. * Changing logic to AND for cache rebuild. --- .../database_apply/database_apply-drupal8/tasks/main.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 23cb0b3b..3473d9e8 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -22,7 +22,7 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site - when: previous_build_number == 0 or (site.force_install is defined and site.force_install) + when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Fix permissions on Drupal directory. shell: @@ -30,12 +30,14 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site - when: previous_build_number == 0 or (site.force_install is defined and site.force_install) + when: (previous_build_number == 0) or (site.force_install is defined and site.force_install) - name: Clear the cache. ansible.builtin.include_role: name: "cache_clear/cache_clear-{{ project_type }}" - when: previous_build_number > 0 + when: + - previous_build_number > 0 + - site.config_import_command != 'deploy' - name: Apply Drupal database updates. command: @@ -46,6 +48,7 @@ with_items: "{{ drupal.sites }}" loop_control: loop_var: site + when: site.config_import_command != 'deploy' - name: Import configuration. command: From 58069b60c45aef39a655e2d0976b123af5a82fbf Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 16:08:26 +0100 Subject: [PATCH 31/40] Attempt to clear the opcache during Drupal deployments. (#182) Co-authored-by: Emlyn Kinzett --- roles/_meta/deploy-drupal7/tasks/main.yml | 2 ++ roles/_meta/deploy-drupal8/tasks/main.yml | 2 ++ roles/database_apply/database_apply-drupal7/tasks/main.yml | 5 +++++ roles/database_apply/database_apply-drupal8/tasks/main.yml | 5 +++++ 4 files changed, 14 insertions(+) diff --git a/roles/_meta/deploy-drupal7/tasks/main.yml b/roles/_meta/deploy-drupal7/tasks/main.yml index 93e6b1b7..cecf3ea0 100644 --- a/roles/_meta/deploy-drupal7/tasks/main.yml +++ b/roles/_meta/deploy-drupal7/tasks/main.yml @@ -24,6 +24,8 @@ name: sanitize/admin_creds - import_role: name: live_symlink +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal7 - import_role: diff --git a/roles/_meta/deploy-drupal8/tasks/main.yml b/roles/_meta/deploy-drupal8/tasks/main.yml index 50804566..6ce31d4f 100644 --- a/roles/_meta/deploy-drupal8/tasks/main.yml +++ b/roles/_meta/deploy-drupal8/tasks/main.yml @@ -26,6 +26,8 @@ name: sanitize/admin_creds - import_role: name: live_symlink +- import_role: + name: cache_clear/cache_clear-opcache - import_role: name: cache_clear/cache_clear-drupal8 - import_role: diff --git a/roles/database_apply/database_apply-drupal7/tasks/main.yml b/roles/database_apply/database_apply-drupal7/tasks/main.yml index eb2edbfa..fe25bcaa 100644 --- a/roles/database_apply/database_apply-drupal7/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal7/tasks/main.yml @@ -24,6 +24,11 @@ name: "cache_clear/cache_clear-{{ project_type }}" when: previous_build_number > 0 +- name: Clear the opcache. + ansible.builtin.include_role: + name: cache_clear/cache_clear-opcache + when: previous_build_number > 0 + - name: Apply Drupal database updates. shell: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 3473d9e8..6dd13e22 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -39,6 +39,11 @@ - previous_build_number > 0 - site.config_import_command != 'deploy' +- name: Clear the opcache. + ansible.builtin.include_role: + name: cache_clear/cache_clear-opcache + when: previous_build_number > 0 + - name: Apply Drupal database updates. command: cmd: "{{ drush_bin }} -l {{ site.folder }} -y updb" From 3ea0bfe4324ea6b9e7acadebba771b4264a764e4 Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Fri, 9 Dec 2022 18:32:18 +0100 Subject: [PATCH 32/40] Better drush deploy support pr 1.x (#185) * Handling the 'drush deploy' command more elegantly for Drupal 8+. * Changing logic to AND for cache rebuild. * Em's clear cache code needs to run for each 'site' too. --- roles/_init/defaults/main.yml | 2 +- roles/database_apply/database_apply-drupal8/tasks/main.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 790e9409..432485d9 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -10,7 +10,7 @@ drupal: public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" diff --git a/roles/database_apply/database_apply-drupal8/tasks/main.yml b/roles/database_apply/database_apply-drupal8/tasks/main.yml index 6dd13e22..16f0417b 100644 --- a/roles/database_apply/database_apply-drupal8/tasks/main.yml +++ b/roles/database_apply/database_apply-drupal8/tasks/main.yml @@ -35,6 +35,9 @@ - name: Clear the cache. ansible.builtin.include_role: name: "cache_clear/cache_clear-{{ project_type }}" + with_items: "{{ drupal.sites }}" + loop_control: + loop_var: site when: - previous_build_number > 0 - site.config_import_command != 'deploy' From b83534407d373150382dfd431802f5fbf31dc2d4 Mon Sep 17 00:00:00 2001 From: tymofiisobchenko <104431720+tymofiisobchenko@users.noreply.github.com> Date: Fri, 30 Dec 2022 13:22:01 +0200 Subject: [PATCH 33/40] Cron job schedule params pr 1.x (#190) * cron job schedule params, namespaces * drupal7-cron-params * cron-job-schedule-params-fix-defaults --- roles/cron/cron_drupal7/defaults/main.yml | 3 +++ roles/cron/cron_drupal7/tasks/job.yml | 3 +++ roles/cron/cron_drupal8/defaults/main.yml | 4 ++++ roles/cron/cron_drupal8/tasks/cron.yml | 2 +- roles/cron/cron_drupal8/tasks/job.yml | 13 +++++++++---- roles/cron/cron_drupal8/tasks/main.yml | 4 ++-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index a6551ab6..ef3ab79f 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -7,6 +7,9 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index 0f1d3203..dd18979c 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -25,6 +25,9 @@ name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" + day: "{{ entry.day | default(omit) }}" + weekday: "{{ entry.weekday | default(omit) }}" + month: "{{ entry.month | default(omit) }}" job: "{{ _cron_job_command }}" state: present disabled: "{{ entry.disabled | default(omit) }}" diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml index 61e2ef8f..913eb530 100644 --- a/roles/cron/cron_drupal8/defaults/main.yml +++ b/roles/cron/cron_drupal8/defaults/main.yml @@ -7,7 +7,11 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] diff --git a/roles/cron/cron_drupal8/tasks/cron.yml b/roles/cron/cron_drupal8/tasks/cron.yml index 2f0c6116..1ed40cc3 100644 --- a/roles/cron/cron_drupal8/tasks/cron.yml +++ b/roles/cron/cron_drupal8/tasks/cron.yml @@ -1,6 +1,6 @@ --- - name: Setup Drupal cron tasks. - include_tasks: + ansible.builtin.include_tasks: file: job.yml with_items: "{{ site.cron }}" loop_control: diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml index bcdac2af..908abff6 100644 --- a/roles/cron/cron_drupal8/tasks/job.yml +++ b/roles/cron/cron_drupal8/tasks/job.yml @@ -1,10 +1,10 @@ --- - name: Define cron job command. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). - set_fact: + ansible.builtin.set_fact: _cron_job_command: "cd {{ _ce_deploy_base_dir }} && {{ _ce_deploy_ansible_location }} {{ drupal.defer_target }} -m shell -a \"{{ _cron_job_command }}\"" when: - drupal.defer is defined @@ -13,7 +13,7 @@ - drupal.defer_target | length > 0 - name: Define cron job command for differing deploy users. - set_fact: + ansible.builtin.set_fact: _cron_job_command: "{{ _cron_job_command }} --extra-vars '{\"become\":true,\"become_user\":\"{{ www_user }}\"}'" when: - www_user != deploy_user @@ -21,9 +21,14 @@ - drupal.defer - name: Setup Drupal cron tasks on app server. - cron: + ansible.builtin.cron: name: "{{ project_name }}_{{ site.folder }}_{{ build_type }}_{{ entry.job }}" minute: "{{ entry.minute }}" hour: "{{ entry.hour | default(omit) }}" + day: "{{ entry.day | default(omit) }}" + weekday: "{{ entry.weekday | default(omit) }}" + month: "{{ entry.month | default(omit) }}" job: "{{ _cron_job_command }}" + state: present + disabled: "{{ entry.disabled | default(omit) }}" delegate_to: "{{ 'localhost' if drupal.defer else inventory_hostname }}" diff --git a/roles/cron/cron_drupal8/tasks/main.yml b/roles/cron/cron_drupal8/tasks/main.yml index 75fa05c7..cf0f80f6 100644 --- a/roles/cron/cron_drupal8/tasks/main.yml +++ b/roles/cron/cron_drupal8/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Setup PATH in crontab. - cron: + ansible.builtin.cron: name: PATH env: true job: "/usr/bin:/usr/local/bin:/bin:/home/{{ deploy_user }}/.bin" @@ -10,7 +10,7 @@ - drupal.defer - name: Create site cron entries. - include_tasks: + ansible.builtin.include_tasks: file: cron.yml with_items: "{{ drupal.sites }}" loop_control: From e97e852235ff29185e7d52ceffe9c28f8e87a11f Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Mon, 23 Jan 2023 17:21:16 +0100 Subject: [PATCH 34/40] Adding option to stop services that might interfere with a squashfs mount. (#193) --- roles/deploy_code/defaults/main.yml | 5 ++++- roles/deploy_code/tasks/cleanup.yml | 33 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/roles/deploy_code/defaults/main.yml b/roles/deploy_code/defaults/main.yml index f1f7cdb4..f9b84739 100644 --- a/roles/deploy_code/defaults/main.yml +++ b/roles/deploy_code/defaults/main.yml @@ -25,11 +25,14 @@ deploy_code: # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" - # List of services to reload to free the loop device for 'squashfs' builds, post lazy umount. + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. services: [] # services: # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. rebuild_infra: false # Details of API call to trigger. See api_call role. diff --git a/roles/deploy_code/tasks/cleanup.yml b/roles/deploy_code/tasks/cleanup.yml index 353854b3..a5da0d87 100644 --- a/roles/deploy_code/tasks/cleanup.yml +++ b/roles/deploy_code/tasks/cleanup.yml @@ -144,6 +144,23 @@ loop_var: www_service become: true when: + - deploy_code.service_action == "reload" + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - deploy_code.services | length > 0 + +- name: Stop any services that might be keeping the loop device busy. + ansible.builtin.service: + name: "{{ www_service }}" + state: stopped + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" - deploy_code.mount_sync is defined - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" @@ -169,6 +186,22 @@ - deploy_code.mount_sync | length > 1 - deploy_code.mount_type == "squashfs" +- name: Start any services we stopped. + ansible.builtin.service: + name: "{{ www_service }}" + state: started + with_items: "{{ deploy_code.services }}" + loop_control: + loop_var: www_service + become: true + when: + - deploy_code.service_action == "stop" + - deploy_code.mount_sync is defined + - deploy_code.mount_sync | length > 1 + - deploy_code.mount_type == "squashfs" + - _deploy_code_mount_check is succeeded + - deploy_code.services | length > 0 + - name: Trigger an infrastructure rebuild. ansible.builtin.include_role: name: api_call From 7bbb77fbd07bedcbc9d19f0d5354b03b1b34186b Mon Sep 17 00:00:00 2001 From: Greg Harvey Date: Wed, 25 Jan 2023 15:32:19 +0100 Subject: [PATCH 35/40] Drush refactor pr 1.x (#197) * Updating docs. * Consistent drush handling in cron. * Linting cruft and removing unimplemented drush.yml handling from D7. * Refactor drush role to support new GitHub release format. * Updating drush README. * Casting the first part of the drush version string as an integer. * Refactoring integer handling. * Major version string will always be a string, assuming below 8 will never exist. * Downloaded drush archive is remote, not local. * Refactoring handling of .bin directory. * Supporting drush-launcher. * Removing drush installation support for Drupal 8 and above. * Bad variable name in Drupal cron jobs. * Fixing maintenance mode command linting and consistency. * Moving chdir to args to see if it helps. * Trying an actual 'cd' command inline. * Cannot '&&' in a cmd. * Trying drush's --root flag to set Drupal path. * With --root we do not need chdir for drush. * Revert "With --root we do not need chdir for drush." This reverts commit d07aaf798a8f5699ed669f9411db2810e2bcd9d8. --- docs/_Sidebar.md | 2 + docs/roles/_init.md | 6 ++- docs/roles/_meta/deploy-simplesamlphp.md | 9 +++++ docs/roles/api_call.md | 26 +++++++++++++ docs/roles/cli/cachetool.md | 2 +- docs/roles/cli/drush.md | 10 +++-- .../cron_database_backup-mysql.md | 1 + docs/roles/cron/cron_drupal7.md | 6 ++- docs/roles/cron/cron_drupal8.md | 7 +++- docs/roles/database_backup.md | 2 +- .../database_backup/database_backup-mysql.md | 1 + docs/roles/deploy_code.md | 37 ++++++++++++++++++- .../sync/database_sync/database_sync-mysql.md | 10 +++++ roles/_init/README.md | 6 ++- roles/_init/defaults/main.yml | 1 + roles/_init/tasks/drupal7.yml | 2 +- roles/_init/tasks/drupal8.yml | 6 +-- roles/api_call/README.md | 17 +++++++++ roles/cli/cachetool/README.md | 2 +- roles/cli/drush/README.md | 10 +++-- roles/cli/drush/defaults/main.yml | 5 +-- roles/cli/drush/tasks/main.yml | 11 +++--- .../config_generate-drupal7/tasks/drush.yml | 10 ----- .../config_generate-drupal7/tasks/main.yml | 10 +---- .../config_generate-drupal8/tasks/drush.yml | 3 +- .../config_generate-drupal8/tasks/main.yml | 4 +- .../config_generate-matomo/tasks/main.yml | 3 +- .../config_generate-mautic/tasks/main.yml | 11 +++--- roles/config_generate/tasks/main.yml | 7 ++-- .../cron_database_backup-mysql/README.md | 1 + roles/cron/cron_drupal7/README.md | 6 ++- roles/cron/cron_drupal7/defaults/main.yml | 1 + roles/cron/cron_drupal7/tasks/job.yml | 2 +- roles/cron/cron_drupal8/README.md | 7 +++- roles/cron/cron_drupal8/defaults/main.yml | 3 +- roles/cron/cron_drupal8/tasks/job.yml | 2 +- roles/database_backup/README.md | 2 +- .../database_backup-mysql/README.md | 1 + roles/deploy_code/README.md | 37 ++++++++++++++++++- .../tasks/main.yml | 2 +- .../tasks/offline.yml | 11 +++--- .../tasks/online.yml | 10 +++-- .../database_sync-mysql/README.md | 10 +++++ 43 files changed, 233 insertions(+), 89 deletions(-) create mode 100644 docs/roles/_meta/deploy-simplesamlphp.md create mode 100644 docs/roles/api_call.md delete mode 100644 roles/config_generate/config_generate-drupal7/tasks/drush.yml diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index efa27c64..16139d8f 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -3,6 +3,7 @@ - [Install](/install) - [Usage](/scripts) - [Roles](roles) + - [API call](/roles/api_call) - [Config](/roles/cache_clear) - [Drupal 7](/roles/cache_clear/cache_clear-drupal7) - [Drupal 8](/roles/cache_clear/cache_clear-drupal8) @@ -37,6 +38,7 @@ - [Drupal 8](/roles/_meta/deploy-drupal8) - [Matomo](/roles/_meta/deploy-matomo) - [Mautic](/roles/_meta/deploy-mautic) + - [SimpleSAMLphp](/roles/_meta/deploy-simplesamlphp) - [NPM](/roles/npm) - [Sync roles](/roles/sync) - [Database sync](/roles/sync/database_sync) diff --git a/docs/roles/_init.md b/docs/roles/_init.md index f733d1a3..1c7e835e 100644 --- a/docs/roles/_init.md +++ b/docs/roles/_init.md @@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least. --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" @@ -32,7 +34,7 @@ drupal: mautic: image_path: "media/images" force_install: false - +bin_directory: "/home/{{ deploy_user }}/.bin" ``` diff --git a/docs/roles/_meta/deploy-simplesamlphp.md b/docs/roles/_meta/deploy-simplesamlphp.md new file mode 100644 index 00000000..3d82e671 --- /dev/null +++ b/docs/roles/_meta/deploy-simplesamlphp.md @@ -0,0 +1,9 @@ +# SimpleSAMLphp +Role for deploying single SimpleSAMLphp instances. Do not use if you are deploying SimpleSAMLphp with another application like Drupal via composer. + +This role currently assumes all config is in the repository alongside composer.json and the special `SIMPLESAMLPHP_CONFIG_DIR` variable is passed in via the web server vhost to tell SimpleSAMLphp where the config is on the server. For vhost configuration in Nginx see ce-provision: + +* https://github.com/codeenigma/ce-provision/blob/1.x/roles/nginx + + + diff --git a/docs/roles/api_call.md b/docs/roles/api_call.md new file mode 100644 index 00000000..15ea2788 --- /dev/null +++ b/docs/roles/api_call.md @@ -0,0 +1,26 @@ +# API call + +Making RESTful API calls to other platforms. + + + + + +## Default variables +```yaml +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 +``` + + diff --git a/docs/roles/cli/cachetool.md b/docs/roles/cli/cachetool.md index 833eb8fc..9cba4a32 100644 --- a/docs/roles/cli/cachetool.md +++ b/docs/roles/cli/cachetool.md @@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: 8.2.2 + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version ``` diff --git a/docs/roles/cli/drush.md b/docs/roles/cli/drush.md index 30f02549..41b673c2 100644 --- a/docs/roles/cli/drush.md +++ b/docs/roles/cli/drush.md @@ -1,13 +1,15 @@ # Drush -Installs the `drush` command-line tool for the deploy user. +Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23). + +For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush` documentation](https://www.drush.org/latest/install/). + ## Default variables ```yaml --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false ``` diff --git a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md index ee71e2bf..4d5bdab5 100644 --- a/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md +++ b/docs/roles/cron/cron_database_backup/cron_database_backup-mysql.md @@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases. cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here ``` diff --git a/docs/roles/cron/cron_drupal7.md b/docs/roles/cron/cron_drupal7.md index 8222ddaf..14356d15 100644 --- a/docs/roles/cron/cron_drupal7.md +++ b/docs/roles/cron/cron_drupal7.md @@ -14,12 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/docs/roles/cron/cron_drupal8.md b/docs/roles/cron/cron_drupal8.md index ae25adc4..bafd49b0 100644 --- a/docs/roles/cron/cron_drupal8.md +++ b/docs/roles/cron/cron_drupal8.md @@ -14,13 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/docs/roles/database_backup.md b/docs/roles/database_backup.md index 947c8053..772efa70 100644 --- a/docs/roles/database_backup.md +++ b/docs/roles/database_backup.md @@ -7,7 +7,7 @@ Generate backups for each build. database_backup: engines: - mysql - + revert: true ``` diff --git a/docs/roles/database_backup/database_backup-mysql.md b/docs/roles/database_backup/database_backup-mysql.md index 5b0d7289..b52413bf 100644 --- a/docs/roles/database_backup/database_backup-mysql.md +++ b/docs/roles/database_backup/database_backup-mysql.md @@ -7,6 +7,7 @@ Generate MySQL backups for each build. mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/docs/roles/deploy_code.md b/docs/roles/deploy_code.md index bad0040f..4e371f28 100644 --- a/docs/roles/deploy_code.md +++ b/docs/roles/deploy_code.md @@ -11,7 +11,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -23,12 +23,45 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 ``` diff --git a/docs/roles/sync/database_sync/database_sync-mysql.md b/docs/roles/sync/database_sync/database_sync-mysql.md index bcb9c6ad..cf7e5311 100644 --- a/docs/roles/sync/database_sync/database_sync-mysql.md +++ b/docs/roles/sync/database_sync/database_sync-mysql.md @@ -5,6 +5,7 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. @@ -20,6 +21,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -30,6 +38,8 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" ``` diff --git a/roles/_init/README.md b/roles/_init/README.md index f733d1a3..1c7e835e 100644 --- a/roles/_init/README.md +++ b/roles/_init/README.md @@ -10,14 +10,16 @@ These variables **must** be set in the `deploy/common.yml` file, at least. --- # Common defaults. Given the "_init" role is mandatory, # this will ensure defaults to other roles too. +# If you are using ce-provision to deploy infrastructure this must match the `user_deploy.username` variable deploy_user: "deploy" +_mysqldump_params: "--max-allowed-packet=128M --single-transaction --skip-opt -e --quick --skip-disable-keys --skip-add-locks -C -a --add-drop-table" drupal: sites: - folder: "default" public_files: "sites/default/files" # Drupal 8 variables config_sync_directory: "config/sync" - config_import_command: "" # i.e. "cim" + config_import_command: "" # i.e. "cim" - set this to "deploy" and cache rebuild and db updates will be skipped # End Drupal 8 variables # Drupal 7 variables revert_features_command: "" # i.e. "fra" @@ -32,7 +34,7 @@ drupal: mautic: image_path: "media/images" force_install: false - +bin_directory: "/home/{{ deploy_user }}/.bin" ``` diff --git a/roles/_init/defaults/main.yml b/roles/_init/defaults/main.yml index 432485d9..b38c1e59 100644 --- a/roles/_init/defaults/main.yml +++ b/roles/_init/defaults/main.yml @@ -25,3 +25,4 @@ drupal: mautic: image_path: "media/images" force_install: false +bin_directory: "/home/{{ deploy_user }}/.bin" \ No newline at end of file diff --git a/roles/_init/tasks/drupal7.yml b/roles/_init/tasks/drupal7.yml index 6487564c..133b3d73 100644 --- a/roles/_init/tasks/drupal7.yml +++ b/roles/_init/tasks/drupal7.yml @@ -1,7 +1,7 @@ --- - name: Define Drush path. set_fact: - drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}" + drush_bin: "{{ drush_bin | default('{{ bin_directory }}/drush.phar') }}" - name: Ensure we have Drush binary. import_role: diff --git a/roles/_init/tasks/drupal8.yml b/roles/_init/tasks/drupal8.yml index 6487564c..ebe63b82 100644 --- a/roles/_init/tasks/drupal8.yml +++ b/roles/_init/tasks/drupal8.yml @@ -1,8 +1,4 @@ --- - name: Define Drush path. set_fact: - drush_bin: "{{ drush_bin | default('/home/{{ deploy_user }}/.bin/drush.phar') }}" - -- name: Ensure we have Drush binary. - import_role: - name: cli/drush + drush_bin: "{{ drush_bin | default('{{ deploy_path }}/vendor/bin/drush') }}" diff --git a/roles/api_call/README.md b/roles/api_call/README.md index 83d1ab28..15ea2788 100644 --- a/roles/api_call/README.md +++ b/roles/api_call/README.md @@ -6,4 +6,21 @@ Making RESTful API calls to other platforms. +## Default variables +```yaml +--- +api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects # see documentation - https://docs.gitlab.com/ee/api/ + method: GET + token: "" # empty means anonymous action + token_type: trigger # options are 'trigger' or 'personal' + variables: [] + status_codes: + - 200 + - 201 + - 202 +``` + diff --git a/roles/cli/cachetool/README.md b/roles/cli/cachetool/README.md index 833eb8fc..9cba4a32 100644 --- a/roles/cli/cachetool/README.md +++ b/roles/cli/cachetool/README.md @@ -5,7 +5,7 @@ Installs the `drush` command-line tool for the deploy user. ```yaml --- cachetool: - version: 8.2.2 + version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version ``` diff --git a/roles/cli/drush/README.md b/roles/cli/drush/README.md index 30f02549..41b673c2 100644 --- a/roles/cli/drush/README.md +++ b/roles/cli/drush/README.md @@ -1,13 +1,15 @@ # Drush -Installs the `drush` command-line tool for the deploy user. +Installs the `drush` command-line tool for the deploy user. **This only works with Drupal 7** and will be withdrawn with [Drupal 7 EOL](https://www.drupal.org/psa-2022-02-23). + +For Drupal 8 and above you must install `drush` with `composer` [as described in the `drush` documentation](https://www.drush.org/latest/install/). + ## Default variables ```yaml --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false ``` diff --git a/roles/cli/drush/defaults/main.yml b/roles/cli/drush/defaults/main.yml index 68ff2c66..746fce44 100644 --- a/roles/cli/drush/defaults/main.yml +++ b/roles/cli/drush/defaults/main.yml @@ -1,6 +1,5 @@ --- drush: - # Note: This is the "default" version, - # but projects should define theirs in composer.json. - version: 8.2.2 + # Where possible always load drush in your Drupal website with composer. + version: 8.4.11 use_vendor: false \ No newline at end of file diff --git a/roles/cli/drush/tasks/main.yml b/roles/cli/drush/tasks/main.yml index 4d27aa3e..5c73be51 100644 --- a/roles/cli/drush/tasks/main.yml +++ b/roles/cli/drush/tasks/main.yml @@ -1,6 +1,6 @@ --- - name: Check if we already have a global Drush. - stat: + ansible.builtin.stat: path: "{{ drush_bin }}" register: drush_global when: @@ -16,18 +16,19 @@ register: drush_global_directory - name: Check if installed Drush version match. - shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" + ansible.builtin.shell: "{{ drush_bin }} --version | grep -o '[0-9]\\.[0-9]\\.[0-9]'" register: drush_global_version when: - deploy_operation == 'deploy' - drush_global.stat.exists - not drush.use_vendor -- name: Download Drush installer. - get_url: +- name: Download Drush installer for Drupal 7. + ansible.builtin.get_url: url: "https://github.com/drush-ops/drush/releases/download/{{ drush.version }}/drush.phar" dest: "{{ drush_bin }}" mode: 0755 + force: true when: - deploy_operation == 'deploy' - not drush.use_vendor @@ -49,7 +50,7 @@ - deploy_operation == 'deploy' - name: Trigger Drush init tasks. - command: "{{ drush_bin }} init -y" + ansible.builtin.command: "{{ drush_bin }} init -y" when: - deploy_operation == 'deploy' - not drush.use_vendor diff --git a/roles/config_generate/config_generate-drupal7/tasks/drush.yml b/roles/config_generate/config_generate-drupal7/tasks/drush.yml deleted file mode 100644 index 9c85eb9d..00000000 --- a/roles/config_generate/config_generate-drupal7/tasks/drush.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -- name: Generates drush.yml file. - template: - src: '{{ item }}' - dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml" - with_first_found: - - "{{ playbook_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml.j2" - - "{{ _ce_deploy_build_dir }}/{{ webroot }}/sites/{{ site.folder }}/{{ build_type }}.drush.yml" - - "drush.yml.j2" \ No newline at end of file diff --git a/roles/config_generate/config_generate-drupal7/tasks/main.yml b/roles/config_generate/config_generate-drupal7/tasks/main.yml index b984fd58..23781ded 100644 --- a/roles/config_generate/config_generate-drupal7/tasks/main.yml +++ b/roles/config_generate/config_generate-drupal7/tasks/main.yml @@ -1,14 +1,6 @@ --- - name: Generates settings file. - include_tasks: settings.yml + ansible.builtin.include_tasks: settings.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site - -#- name: Generates Drush settings file. -# include_tasks: drush.yml -# when: -# - site.base_url is defined -# with_items: "{{ drupal.sites }}" -# loop_control: -# loop_var: site diff --git a/roles/config_generate/config_generate-drupal8/tasks/drush.yml b/roles/config_generate/config_generate-drupal8/tasks/drush.yml index 9c85eb9d..f4b7ae7a 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/drush.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/drush.yml @@ -1,7 +1,6 @@ --- - - name: Generates drush.yml file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}/drush.yml" with_first_found: diff --git a/roles/config_generate/config_generate-drupal8/tasks/main.yml b/roles/config_generate/config_generate-drupal8/tasks/main.yml index cd3c19fd..3db5bf60 100644 --- a/roles/config_generate/config_generate-drupal8/tasks/main.yml +++ b/roles/config_generate/config_generate-drupal8/tasks/main.yml @@ -1,12 +1,12 @@ --- - name: Generates settings file. - include_tasks: settings.yml + ansible.builtin.include_tasks: settings.yml with_items: "{{ drupal.sites }}" loop_control: loop_var: site - name: Generates Drush settings file. - include_tasks: drush.yml + ansible.builtin.include_tasks: drush.yml when: - site.base_url is defined with_items: "{{ drupal.sites }}" diff --git a/roles/config_generate/config_generate-matomo/tasks/main.yml b/roles/config_generate/config_generate-matomo/tasks/main.yml index 505ff3ce..e5482e65 100644 --- a/roles/config_generate/config_generate-matomo/tasks/main.yml +++ b/roles/config_generate/config_generate-matomo/tasks/main.yml @@ -1,7 +1,6 @@ --- - - name: Generates config file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/config/config.ini.php" with_first_found: diff --git a/roles/config_generate/config_generate-mautic/tasks/main.yml b/roles/config_generate/config_generate-mautic/tasks/main.yml index d1da20c5..2a9d63d0 100644 --- a/roles/config_generate/config_generate-mautic/tasks/main.yml +++ b/roles/config_generate/config_generate-mautic/tasks/main.yml @@ -1,15 +1,14 @@ --- - -- name: set public files path. - set_fact: +- name: Set public files path. + ansible.builtin.set_fact: build_image_path: "{{ mautic.image_path }}" -- name: set private files path. - set_fact: +- name: Set private files path. + ansible.builtin.set_fact: build_upload_dir: "{{ deploy_assets_base_path }}/{{ project_name }}_{{ build_type }}_private_files" - name: Generates local.php file. - template: + ansible.builtin.template: src: '{{ item }}' dest: "{{ deploy_path }}/{{ webroot }}/app/config/local.php" with_first_found: diff --git a/roles/config_generate/tasks/main.yml b/roles/config_generate/tasks/main.yml index 3bce314b..2af08e87 100644 --- a/roles/config_generate/tasks/main.yml +++ b/roles/config_generate/tasks/main.yml @@ -1,12 +1,11 @@ --- - - name: Generate project specific configuration. - include_role: + ansible.builtin.include_role: name: "config_generate/config_generate-{{ project_type }}" when: deploy_operation == 'deploy' - name: Generate additional templates. - template: + ansible.builtin.template: src: "{{ template.src }}" dest: "{{ deploy_path }}/{{ template.dest }}" with_items: "{{ config_generate.templates }}" @@ -17,7 +16,7 @@ - deploy_operation == 'deploy' - name: Create additional symlinks. - file: + ansible.builtin.file: src: "{{ link.src }}" dest: "{{ deploy_path }}/{{ link.dest }}" state: link diff --git a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md index ee71e2bf..4d5bdab5 100644 --- a/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md +++ b/roles/cron/cron_database_backup/cron_database_backup-mysql/README.md @@ -7,6 +7,7 @@ Ensure regular local backups of MySQL databases. cron_mysql_backup: dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" keep: 10 + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here ``` diff --git a/roles/cron/cron_drupal7/README.md b/roles/cron/cron_drupal7/README.md index 8222ddaf..14356d15 100644 --- a/roles/cron/cron_drupal7/README.md +++ b/roles/cron/cron_drupal7/README.md @@ -14,12 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/roles/cron/cron_drupal7/defaults/main.yml b/roles/cron/cron_drupal7/defaults/main.yml index ef3ab79f..419e9c4b 100644 --- a/roles/cron/cron_drupal7/defaults/main.yml +++ b/roles/cron/cron_drupal7/defaults/main.yml @@ -16,3 +16,4 @@ drupal: defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" \ No newline at end of file diff --git a/roles/cron/cron_drupal7/tasks/job.yml b/roles/cron/cron_drupal7/tasks/job.yml index dd18979c..b89a0b05 100644 --- a/roles/cron/cron_drupal7/tasks/job.yml +++ b/roles/cron/cron_drupal7/tasks/job.yml @@ -1,7 +1,7 @@ --- - name: Define cron job command. ansible.builtin.set_fact: - _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drush_bin }} {{ entry.job }}" + _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). ansible.builtin.set_fact: diff --git a/roles/cron/cron_drupal8/README.md b/roles/cron/cron_drupal8/README.md index ae25adc4..bafd49b0 100644 --- a/roles/cron/cron_drupal8/README.md +++ b/roles/cron/cron_drupal8/README.md @@ -14,13 +14,16 @@ drupal: cron: # These are the relevant parts for cron. - minute: "*/{{ 10 | random(start=1) }}" # hour: 2 + # day: + # weekday: + # month: job: cron + # disabled: true # If the sites are being deployed to an ASG, setting defer to true will create the crontab entry on the deploy server rather than all of the app servers. defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" ``` diff --git a/roles/cron/cron_drupal8/defaults/main.yml b/roles/cron/cron_drupal8/defaults/main.yml index 913eb530..9d4b90f6 100644 --- a/roles/cron/cron_drupal8/defaults/main.yml +++ b/roles/cron/cron_drupal8/defaults/main.yml @@ -16,5 +16,4 @@ drupal: defer: false # If defer is set to true, the Ansible target must be declared with defer_target. If using a group, include the index. For example, _ce_www_dev[0] defer_target: "" - # Drush location when installed with Composer - drush_location: "vendor/drush/drush/drush" + drush_location: "{{ drush_bin }}" # you might specify another location, e.g. "{{ deploy_path }}/vendor/bin/drush" diff --git a/roles/cron/cron_drupal8/tasks/job.yml b/roles/cron/cron_drupal8/tasks/job.yml index 908abff6..b89a0b05 100644 --- a/roles/cron/cron_drupal8/tasks/job.yml +++ b/roles/cron/cron_drupal8/tasks/job.yml @@ -1,7 +1,7 @@ --- - name: Define cron job command. ansible.builtin.set_fact: - _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ deploy_path }}/{{ drupal.drush_location }} {{ entry.job }}" + _cron_job_command: "cd {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }} && {{ drupal.drush_location }} {{ entry.job }}" - name: Define cron job command if deferred (ASG). ansible.builtin.set_fact: diff --git a/roles/database_backup/README.md b/roles/database_backup/README.md index 947c8053..772efa70 100644 --- a/roles/database_backup/README.md +++ b/roles/database_backup/README.md @@ -7,7 +7,7 @@ Generate backups for each build. database_backup: engines: - mysql - + revert: true ``` diff --git a/roles/database_backup/database_backup-mysql/README.md b/roles/database_backup/database_backup-mysql/README.md index 5b0d7289..b52413bf 100644 --- a/roles/database_backup/database_backup-mysql/README.md +++ b/roles/database_backup/database_backup-mysql/README.md @@ -7,6 +7,7 @@ Generate MySQL backups for each build. mysql_backup: handling: rolling dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/build" + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here # Number of dumps/db to keep. Note this is independant from the build codebases. keep: 10 # This can be one of the following: diff --git a/roles/deploy_code/README.md b/roles/deploy_code/README.md index bad0040f..4e371f28 100644 --- a/roles/deploy_code/README.md +++ b/roles/deploy_code/README.md @@ -11,7 +11,7 @@ deploy_code: # Specify any additional symlink to create, with src (target) and dest (link). # src: can be either absolute or relative to the dest (eg. '/var/my_data', '/home/deploy/simplesaml', '../../../myconfig') # dest: can only be relative to the root of your repository (eg. 'www/themes/myassets', 'var/cache') - # create: wether to create the target if it does not exists. + # create: whether to create the target if it does not exists. # - src: '/home/{{ deploy_user }}//{{ project_name }}_{{ build_type }}/simplesaml' # dest: 'vendor/simplesamlphp/simplesamlphp/config' # - src: '/var/simplesaml/etc' @@ -23,12 +23,45 @@ deploy_code: templates: [] # Number of builds to keep. Note this is independant of databases/dump. keep: 10 - # Wether to sync the local deploy base to a shared destination, after successful build. + # Whether to sync the local deploy base to a shared destination, after successful build. mount_sync: "" # mount_sync: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/deploy" + # Type of file to use for sync - 'squashfs' or 'tarball' + # @see the _init role for SquashFS build dir paths + # @see the squashfs role in ce-provision which installs the special conditions required by the deploy user to use this behaviour + mount_type: "tarball" # Path that you want to make sure has 755 permissions. Make sure to include the webroot WITHOUT the slash. perms_fix_path: "" # perms_fix_path: "www/sites/default" + # List of services to manipulate to free the loop device for 'squashfs' builds, post lazy umount. + # @see the squashfs role in ce-provision where special permissions for deploy user to manipulate services get granted. + services: [] + # services: + # - php8.0-fpm + # What action to take against the services, 'reload' or 'stop'. + # Busy websites will require a hard stop of services to achieve the umount command. + service_action: reload + # Trigger an API call to rebuild infra after a deploy, e.g. if you need to repack an AMI. + rebuild_infra: false + # Details of API call to trigger. See api_call role. + api_call: + type: gitlab + base_url: https://gitlab.example.com/api/v4/ + path: projects/1/ref/main/trigger/pipeline + method: POST + token: asdf-1234 + token_type: trigger + variables: [] + # example build parameters + # - "[ENV]=dev" + # - "[PLAY]=myserver.yml" + # - "[RESOURCE]=myserver-example-com" + # - "[REGION]=eu-west-1" + # - "[EXTRA_PARAMS]=--force" + status_codes: + - 200 + - 201 + - 202 ``` diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml index 0633bc75..ccc2c2cf 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/main.yml @@ -1,5 +1,5 @@ --- -- include_tasks: "{{ maintenance_mode.operation }}.yml" +- ansible.builtin.include_tasks: "{{ maintenance_mode.operation }}.yml" with_items: "{{ drupal.sites }}" loop_control: loop_var: site diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index cbe65b07..5fd4ddd0 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -1,8 +1,8 @@ --- -# @todo this needs refactoring for multisite. - name: Enable maintenance mode. - command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 1 --input-format=integer --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -11,8 +11,9 @@ - project_type == 'drupal8' - name: Enable maintenance mode D7. - shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index e6a9cbf5..8337c968 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -1,7 +1,8 @@ --- - name: Disable maintenance mode. - command: - cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} state:set system.maintenance_mode 0 --input-format=integer --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" @@ -9,8 +10,9 @@ - project_type == 'drupal8' - name: Disable maintenance mode D7. - shell: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" + ansible.builtin.command: + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" become_user: "{{ www_user }}" diff --git a/roles/sync/database_sync/database_sync-mysql/README.md b/roles/sync/database_sync/database_sync-mysql/README.md index bcb9c6ad..cf7e5311 100644 --- a/roles/sync/database_sync/database_sync-mysql/README.md +++ b/roles/sync/database_sync/database_sync-mysql/README.md @@ -5,6 +5,7 @@ Sync MySQL databases between environments. ```yaml --- mysql_sync: + mysqldump_params: "{{ _mysqldump_params }}" # set in _init but you can override here databases: - source: # Name of the database to take a dump from. @@ -20,6 +21,13 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuildprod + # Whether or not use to create a fresh database backup or use a nightly one. + fresh_db: true + # Location where nightly backups are kept. This must match the value set for cron_mysql_backup.dumps_directory. Below is the default. + # This var is only used when fresh_db is set to "false". + dumps_directory: "/home/{{ deploy_user }}/shared/{{ project_name }}_{{ build_type }}/db_backups/mysql/regular" + # If the source is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" target: database: "{{ project_name }}_dev" credentials_file: "/home/{{ deploy_user }}/.mysql.creds" @@ -30,6 +38,8 @@ mysql_sync: type: fixed # For "rolling builds", so we can compute the database name. build_id: mybuilddev + # If the target is on an ASG, provide the ASG name here. Otherwise, leave empty. + asg: "" ``` From 1f858a6a8acc03bdd9f74c4204e80cb6ec05658e Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 27 Jan 2023 12:53:06 +0200 Subject: [PATCH 36/40] cachetool_settings_and_drupal7_tasks_fix --- .../cache_clear-opcache/defaults/main.yml | 2 +- .../cache_clear-opcache/tasks/main.yml | 6 +- roles/cli/cachetool/defaults/main.yml | 2 +- roles/cli/cachetool/tasks/main.yml | 65 +++++++++++++++++-- .../tasks/offline.yml | 2 +- .../tasks/online.yml | 2 +- 6 files changed, 68 insertions(+), 11 deletions(-) diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 8121226e..5c221989 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -4,7 +4,7 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - adapter: "" + adapter: "127.0.0.1:90{{ _php_version | replace('.','') }}" # Bins to clear. clear_opcache: true clear_apcu: false diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index bee9e6fa..a392509a 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -1,4 +1,8 @@ --- +- name: Get latest php installed + ansible.builtin.shell: + cmd: "ls /etc/php/ | tail -1" + register: _php_version - name: Clear opcache. command: @@ -13,4 +17,4 @@ - name: Clear stats. command: cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n stat:clear" - when: cache_clear_opcache.clear_stat \ No newline at end of file + when: cache_clear_opcache.clear_stat diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index 5ec1b2d7..791253f6 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: latest # # enter three-digit version number, e.g. "7.0.0", to install a specific version \ No newline at end of file + version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. \ No newline at end of file diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index aa27df04..52ec81b9 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -13,15 +13,67 @@ when: - deploy_operation == 'deploy' -- name: Download latest cachetool installer. - get_url: - url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" - dest: "{{ cachetool_bin }}" - mode: 0755 +- name: Download cachetool depending on latest php version installed. # If not specified manually, according to https://github.com/gordalina/cachetool#compatibility + block: + - name: Get latest php installed + ansible.builtin.shell: + cmd: "ls /etc/php/ | tail -1 | sed -e 's/\\.//g'" + register: _php_version_str + + - name: Download latest cachetool installer if PHP is 8.1 or newer. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int >= 81 + + - name: Download cachetool version 8.5.0 installer if PHP is 8.0. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 80 + + - name: Download cachetool version 7.1.0 installer if PHP is 7.3 or newer. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int >= 73 + - _php_version_str.stdout | int < 80 + + - name: Download cachetool version 5.1.3 installer if PHP is 7.2. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 72 + + - name: Download cachetool version 4.1.1 installer if PHP is 7.1. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int == 71 + + - name: Download cachetool version 3.2.2 installer if PHP version is too old. + get_url: + url: "http://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar" + dest: "{{ cachetool_bin }}" + mode: 0755 + when: + - _php_version_str.stdout | int < 71 + when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - cachetool.version == 'latest' + - cachetool.version is not defined - name: "Download cachetool version {{ cachetool.version }} installer." get_url: @@ -31,4 +83,5 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - - cachetool.version != 'latest' + - cachetool.version is defined + - cachetool.version | length > 0 diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml index 5fd4ddd0..f8ae276c 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/offline.yml @@ -12,7 +12,7 @@ - name: Enable maintenance mode D7. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1 --root {{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 1" args: chdir: "{{ live_symlink_dest }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" diff --git a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml index 8337c968..dfc3e3b3 100644 --- a/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml +++ b/roles/maintenance_mode/maintenance_mode-drupal-core/tasks/online.yml @@ -11,7 +11,7 @@ - name: Disable maintenance mode D7. ansible.builtin.command: - cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0 --root {{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" + cmd: "{{ drush_bin }} -l {{ site.folder }} vset maintenance_mode 0" args: chdir: "{{ deploy_path }}/{{ webroot }}/sites/{{ site.folder }}" become: "{{ 'no' if www_user == deploy_user else 'yes' }}" From 00aaf0176ee5a6e638dce476a1d869d3c1950073 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 27 Jan 2023 12:55:38 +0200 Subject: [PATCH 37/40] cachetool_settings_and_drupal7_tasks_fix --- roles/cli/cachetool/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cli/cachetool/defaults/main.yml b/roles/cli/cachetool/defaults/main.yml index 791253f6..8b98934d 100644 --- a/roles/cli/cachetool/defaults/main.yml +++ b/roles/cli/cachetool/defaults/main.yml @@ -1,3 +1,3 @@ --- cachetool: - version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. \ No newline at end of file + version: "" # enter three-digit version number, e.g. "7.0.0", to install a specific version. If not specified, will be installed depending on the php version. From 936bb1b37fa2cecb9249dff34c74bab63d3e70ca Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 27 Jan 2023 12:59:52 +0200 Subject: [PATCH 38/40] cachetool_settings_and_drupal7_tasks_fix --- roles/cli/cachetool/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 52ec81b9..2460e6f6 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -72,7 +72,6 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - - cachetool.version == 'latest' - cachetool.version is not defined - name: "Download cachetool version {{ cachetool.version }} installer." From 6d72d55f953fe80fb86f80200cf672403845a4b7 Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 27 Jan 2023 18:16:40 +0200 Subject: [PATCH 39/40] cachetool settings fixing condition and namespace --- roles/cli/cachetool/tasks/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/cli/cachetool/tasks/main.yml b/roles/cli/cachetool/tasks/main.yml index 2460e6f6..12cba942 100644 --- a/roles/cli/cachetool/tasks/main.yml +++ b/roles/cli/cachetool/tasks/main.yml @@ -21,7 +21,7 @@ register: _php_version_str - name: Download latest cachetool installer if PHP is 8.1 or newer. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -29,7 +29,7 @@ - _php_version_str.stdout | int >= 81 - name: Download cachetool version 8.5.0 installer if PHP is 8.0. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-8.5.0.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -37,7 +37,7 @@ - _php_version_str.stdout | int == 80 - name: Download cachetool version 7.1.0 installer if PHP is 7.3 or newer. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-7.1.0.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -46,7 +46,7 @@ - _php_version_str.stdout | int < 80 - name: Download cachetool version 5.1.3 installer if PHP is 7.2. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-5.1.3.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -54,7 +54,7 @@ - _php_version_str.stdout | int == 72 - name: Download cachetool version 4.1.1 installer if PHP is 7.1. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-4.1.1.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -62,7 +62,7 @@ - _php_version_str.stdout | int == 71 - name: Download cachetool version 3.2.2 installer if PHP version is too old. - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-3.2.2.phar" dest: "{{ cachetool_bin }}" mode: 0755 @@ -72,10 +72,10 @@ when: - deploy_operation == 'deploy' - not cachetool_global.stat.exists - - cachetool.version is not defined + - cachetool.version | length == 0 - name: "Download cachetool version {{ cachetool.version }} installer." - get_url: + ansible.builtin.get_url: url: "http://gordalina.github.io/cachetool/downloads/cachetool-{{ cachetool.version }}.phar" dest: "{{ cachetool_bin }}" mode: 0755 From 848729b1a0dbcd04747b27d0dfb3a43a602c00bc Mon Sep 17 00:00:00 2001 From: tim Date: Fri, 27 Jan 2023 19:02:49 +0200 Subject: [PATCH 40/40] fixing cachetool adapter options --- .../cache_clear/cache_clear-opcache/defaults/main.yml | 2 +- roles/cache_clear/cache_clear-opcache/tasks/main.yml | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/roles/cache_clear/cache_clear-opcache/defaults/main.yml b/roles/cache_clear/cache_clear-opcache/defaults/main.yml index 5c221989..97623e62 100644 --- a/roles/cache_clear/cache_clear-opcache/defaults/main.yml +++ b/roles/cache_clear/cache_clear-opcache/defaults/main.yml @@ -4,7 +4,7 @@ cache_clear_opcache: # eg. # --fcgi=127.0.0.1:9000 # Leave blank to use /etc/cachetool.yml - adapter: "127.0.0.1:90{{ _php_version | replace('.','') }}" + # adapter: "127.0.0.1:9081" # Leave commented to automatically detect the adapter based on PHP version. # Bins to clear. clear_opcache: true clear_apcu: false diff --git a/roles/cache_clear/cache_clear-opcache/tasks/main.yml b/roles/cache_clear/cache_clear-opcache/tasks/main.yml index a392509a..d7928e9b 100644 --- a/roles/cache_clear/cache_clear-opcache/tasks/main.yml +++ b/roles/cache_clear/cache_clear-opcache/tasks/main.yml @@ -4,17 +4,21 @@ cmd: "ls /etc/php/ | tail -1" register: _php_version +- name: Set cachetool adapter. + ansible.builtin.set_fact: + _cachetool_adapter: "{{ cache_clear_opcache.adapter | default('127.0.0.1:90' + _php_version | replace('.','')) }}" + - name: Clear opcache. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n opcache:reset" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n opcache:reset" when: cache_clear_opcache.clear_opcache - name: Clear apcu. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n apcu:cache:clear all" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n apcu:cache:clear all" when: cache_clear_opcache.clear_apcu - name: Clear stats. command: - cmd: "{{ cachetool_bin }} {{ cache_clear_opcache.adapter }} -n stat:clear" + cmd: "{{ cachetool_bin }} {{ _cachetool_adapter }} -n stat:clear" when: cache_clear_opcache.clear_stat