Skip to content

Commit

Permalink
Tune-up the galera role for efficiency
Browse files Browse the repository at this point in the history
The galera server role has quite a bit going on within it and because of
recent improvements in Ansible we can make better use of tasks, blocks,
facts, local facts, and organization. This change tunes the role up
following some of our better/more modern patterns allowing the role to
not only be more efficient but also easier to understand and improves
the roles idempotency.

Change-Id: If189a8192f22aafb168587361ca8e6903c918697
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
  • Loading branch information
cloudnull committed Nov 29, 2017
1 parent 94821f8 commit ca054bd
Show file tree
Hide file tree
Showing 31 changed files with 570 additions and 950 deletions.
32 changes: 13 additions & 19 deletions defaults/main.yml
Expand Up @@ -21,14 +21,11 @@ galera_cluster_members: "{{ groups['galera_all'] }}"
galera_server_bootstrap_node: "{{ galera_cluster_members[0] }}"
galera_ignore_cluster_state: false
galera_upgrade: false
galera_force_bootstrap: false

galera_wsrep_node_name: "{{ inventory_hostname }}"
galera_cluster_name: openstack_galera_cluster

# This variable will prevent the galera_cluster_name from changing unintentionally on a running cluster.
# To intentionally change the galera_cluster_name set this variable to True
galera_force_change_cluster_name: False

# The galera server-id should be set on all cluster nodes to ensure
# that replication is handled correctly and the error
# "Warning: You should set server-id to a non-0 value if master_host is
Expand Down Expand Up @@ -64,9 +61,6 @@ galera_percona_xtrabackup_repo: "{{ _galera_percona_xtrabackup_repo | default({}
# Enable the use of the upstream percona repo
use_percona_upstream: "{{ _use_percona_upstream }}"

galera_existing_cluster: true
galera_running_and_bootstrapped: false

galera_monitoring_user: monitoring
galera_monitoring_user_password: ""
# NOTE(cloudnull): Set an interface or CIDR to limit the traffic source when
Expand Down Expand Up @@ -98,6 +92,7 @@ galera_innodb_log_buffer_size: 128M

## wsrep configuration
galera_wsrep_address: "{{ ansible_host }}"
galera_wsrep_address_port: "{{ galera_wsrep_address }}:3306"
galera_wsrep_cluster_address: >-
{% set _var = [] -%}
{% for cluster_host in galera_cluster_members -%}
Expand Down Expand Up @@ -181,11 +176,12 @@ galera_use_ssl: false
galera_ssl_cert: /etc/mysql/ssl/galera.pem
galera_ssl_key: /etc/mysql/ssl/galera.key
galera_ssl_ca_cert: /etc/mysql/ssl/galera-ca.pem
# These options should be specified in user_variables if necessary, otherwise self-signed certs are used.
galera_user_ssl_cert: /etc/openstack_deploy/self_signed_certs/galera.pem
galera_user_ssl_key: /etc/openstack_deploy/self_signed_certs/galera.key
galera_user_ssl_ca_cert: /etc/openstack_deploy/self_signed_certs/galera-ca.pem
# Set galera_ssl_self_signed_regen to true if you want to generate a new
## These options should be specified in user_variables if necessary, otherwise self-signed certs are used.
# galera_user_ssl_cert: /etc/openstack_deploy/self_signed_certs/galera.pem
# galera_user_ssl_key: /etc/openstack_deploy/self_signed_certs/galera.key
# galera_user_ssl_ca_cert: /etc/openstack_deploy/self_signed_certs/galera-ca.pem

## Set galera_ssl_self_signed_regen to true if you want to generate a new
# SSL certificate for Galera when this playbook runs. You can also change
# the subject of the self-signed certificate here if you prefer.
galera_ssl_self_signed_regen: false
Expand All @@ -196,20 +192,18 @@ galera_ssl_ca_self_signed_subject: "/C=US/ST=Texas/L=San Antonio/O=IT"
galera_ssl_address: "{{ ansible_host }}"

# MariaDB 10.1+ ships with 'PrivateDevices=True' in the systemd unit file. This
# provides some additional security, but it causes problems with creating
# mount namespaces on CentOS 7 with systemd 219. While the security
# enhancements are helpful on bare metal hosts with multiple services running,
# they are not as helpful when MariaDB is running in a container with its own
# isolated namespaces.
# provides some additional security, but it causes problems with systemd 219.
# While the security enhancements are helpful on bare metal hosts with multiple
# services running, they are not as helpful when MariaDB is running in a
# container with its own isolated namespaces.
#
# Related bugs:
# https://bugs.launchpad.net/openstack-ansible/+bug/1697531
# https://github.com/lxc/lxc/issues/1623
# https://github.com/systemd/systemd/issues/6121
#
# Setting the following variable to 'yes' will disable the PrivateDevices
# setting in the systemd unit file for MariaDB on CentOS 7 hosts.
galera_disable_privatedevices: no
galera_disable_privatedevices: "{{ _galera_disable_privatedevices }}"

## Set default mirror for openSUSE repositories
# NOTE(hwoarang): Ensure that the full path to the 'opensuse' directory is used.
Expand Down
72 changes: 72 additions & 0 deletions files/galera_new_cluster
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

## Shell Opts ----------------------------------------------------------------
set -e

## Functions -----------------------------------------------------------------
function cleanup {
echo "Running Cleanup."
systemctl unset-environment _WSREP_NEW_CLUSTER || systemctl set-environment _WSREP_NEW_CLUSTER=''
if [[ -f "/etc/my.cnf.d/99_bootstrap.cnf" ]]; then
rm /etc/my.cnf.d/99_bootstrap.cnf
fi
}

function wait_operational {
WAITCMD="while ! mysql --silent --skip-column-names -e 'SHOW STATUS LIKE \"wsrep_evs_state\"' | grep -wq \"OPERATIONAL\"; do sleep 5; done"
if ! timeout 180 sh -c "${WAITCMD}"; then
echo "Cluster failed to return an \"OPERATIONAL\" state"
return 1
else
echo "Cluster is now OPERATIONAL"
return 0
fi
}

function bootstrap_opts {
cat > /etc/my.cnf.d/99_bootstrap.cnf <<EOF
[mysqld]
wsrep_new_cluster
EOF
}

## Main ----------------------------------------------------------------------
trap cleanup EXIT INT TERM

EXIT_CODE=0
if ! systemctl status mysql > /dev/null; then
systemctl set-environment _WSREP_NEW_CLUSTER='--wsrep-new-cluster'
if grep -rniq -e suse -e opensuse /etc/os-release; then
bootstrap_opts
fi
if systemctl start mysql; then
EXIT_CODE=3
else
echo "Cluster bootstrap failed."
systemctl status mysql
exit 99
fi
fi

wait_operational

cat <<EOF
NOTICE: Exit code 0 and 3 are success.
Exit 0 is no change, exit 3 is change.
Current Exit Code "${EXIT_CODE}"
EOF

exit ${EXIT_CODE}
123 changes: 54 additions & 69 deletions handlers/main.yml
Expand Up @@ -14,91 +14,76 @@
# limitations under the License.

- name: Reload the systemd daemon
command: "systemctl daemon-reload"
when:
- ansible_service_mgr == 'systemd'

- name: Restart mysql
service:
systemd:
daemon_reload: yes
name: mysql
state: restarted
sleep: 2
arguments: "{{ (not galera_existing_cluster | bool and inventory_hostname == galera_server_bootstrap_node) | ternary('--wsrep-new-cluster', '') }}"
environment:
MYSQLD_STARTUP_TIMEOUT: 180
when:
- not galera_running_and_bootstrapped | bool
- ansible_pkg_mgr != "zypper"
register: galera_restart
until: galera_restart | success
retries: 3
delay: 5
# notifies are only fired when status is "changed"
changed_when: galera_restart | failed
enabled: "yes"

- name: Check node status
command: >
mysql --silent --skip-column-names --connect-timeout=10 -e 'SHOW STATUS LIKE "wsrep_local_state";'
failed_when: false
changed_when: true
register: node_status
listen: Restart all mysql
notify:
- "Remove stale .sst"
- "Restart mysql fall back"
listen: "Restart all mysql"
- Bootstrap cluster
- Restart mysql (All)

- name: Check if node is in the cluster
command: >
mysql --silent --skip-column-names --connect-timeout=10 -e 'SHOW STATUS LIKE "wsrep_incoming_addresses";'
failed_when: false
changed_when: false
register: incoming_addresses
listen: Bootstrap cluster

- name: Remove stale .sst
file:
path: "/var/lib/mysql/.sst"
state: absent
- name: Set incoming addresses fact (primary)
set_fact:
galera_incoming_addresses: "{{ (incoming_addresses.stdout.split()[-1] | default('')).split(',') }}"
listen: Bootstrap cluster

- name: Restart mysql fall back
- name: Set node status fact
set_fact:
galera_cluster_ready: "{{ (galera_wsrep_address_port in galera_incoming_addresses) or ((node_status.stdout.split()[-1] | default(false)) in ['2', '4']) }}"
listen: Bootstrap cluster

- name: Stop mysql
service:
name: mysql
state: restarted
sleep: 2
arguments: "{{ (not galera_existing_cluster | bool and inventory_hostname == galera_server_bootstrap_node) | ternary('--wsrep-new-cluster', '') }}"
environment:
MYSQLD_STARTUP_TIMEOUT: 180
register: galera_restart_fall_back
until: galera_restart_fall_back | success
retries: 3
delay: 5

- name: "Set wsrep-new-cluster"
lineinfile:
line: 'wsrep-new-cluster'
insertafter: 'wsrep_cluster_name'
state: present
dest: "{{ galera_etc_include_dir }}/cluster.cnf"
listen: "Restart all mysql"
state: stopped
changed_when: true
listen: Bootstrap cluster
when:
- not galera_running_and_bootstrapped | bool
- not galera_existing_cluster | bool and inventory_hostname == galera_server_bootstrap_node
- ansible_pkg_mgr == "zypper"
- not galera_cluster_ready | bool

- name: "Ensure node is not in bootstrap mode"
lineinfile:
line: 'wsrep-new-cluster'
state: absent
dest: "{{ galera_etc_include_dir }}/cluster.cnf"
listen: "Restart all mysql"
- name: Start new cluster
command: /usr/local/bin/galera_new_cluster
failed_when: not start_new_cluster.rc in [0, 3]
changed_when: start_new_cluster.rc == 3
register: start_new_cluster
delegate_to: "{{ galera_server_bootstrap_node }}"
run_once: true
when:
- not galera_running_and_bootstrapped | bool
- (not galera_existing_cluster | bool) or (not inventory_hostname == galera_server_bootstrap_node)
- ansible_pkg_mgr == "zypper"
- not galera_cluster_ready | bool
listen: Bootstrap cluster

- name: Restart mysql on SUSE
- name: Restart mysql (All)
service:
name: mysql
state: restarted
when:
- not galera_running_and_bootstrapped | bool
- ansible_pkg_mgr == "zypper"
state: "{{ (not hostvars[item]['galera_cluster_ready'] | bool) | ternary('started', 'restarted') }}"
environment:
MYSQLD_STARTUP_TIMEOUT: 180
register: galera_restart
until: galera_restart | success
retries: 3
retries: 6
delay: 5
# notifies are only fired when status is "changed"
changed_when: galera_restart | failed
failed_when: false
notify:
- "Remove stale .sst"
listen: "Restart all mysql"
delegate_to: "{{ item }}"
when:
- hostvars[item]['galera_cluster_ready'] is defined
with_items:
- "{{ ansible_play_hosts }}"
run_once: true

- meta: noop
listen: Manage LB
Expand Down
74 changes: 0 additions & 74 deletions library/mysql_status_facts

This file was deleted.

0 comments on commit ca054bd

Please sign in to comment.